Dynamic Accent Duplex 0.2.0: Dynamic ASR real as default ASR (omniASR fallback)

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+samples/01_igbo_greetings_english.mp3 filter=lfs diff=lfs merge=lfs -text
+samples/03_pidgin_bargein.mp3 filter=lfs diff=lfs merge=lfs -text
+samples/04_hausa_greetings_english.mp3 filter=lfs diff=lfs merge=lfs -text
+samples/05_igbo_heavy.mp3 filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,173 @@
+---
+license: apache-2.0
+library_name: pytorch
+language:
+- en
+- pcm
+- ig
+- yo
+- ha
+tags:
+- speech-to-speech
+- full-duplex
+- voice-agent
+- code-switching
+- nigerian-languages
+- asr
+- tts
+- experimental
+pipeline_tag: audio-to-audio
+---
+
+# Dynamic Accent Duplex (experimental)
+
+**An experimental cascaded full-duplex voice agent for Nigerian code-switched speech** (English, Nigerian Pidgin, Igbo, Yoruba, Hausa) by Modular.
+You talk however you naturally mix languages; it answers **in the same mix**. Mostly English with an Igbo greeting gets a reply in mostly English with an Igbo greeting. Pidgin gets Pidgin.
+
+This is **not** a native speech-to-speech model (no Moshi-style joint model). It is a pipeline of permissively licensed components with a language-mix controller in the middle:
+
+```
+mic ─► Silero VAD ─► Dynamic ASR (CTC, streaming partials) ─► per-word language ID + mix estimate
+        │  end-of-turn, barge-in, backchannels                          │  {eng: 0.8, ibo: 0.2, pattern: greeting:ibo}
+        ▼                                                               ▼
+   stop playback ◄── barge-in            Qwen3-30B-A3B reply (3 candidates, inline <eng>/<ibo>/... tags)
+                                                                        ▼
+                                          validator: mix distance, tags, greeting, length, loops, echo ─► regenerate / fallback
+                                                                        ▼
+speaker ◄── Dynamic Accent Flash-DPO TTS (chunk streaming, code-switching voice) ◄──┘
+```
+
+**Status: research demo.** Igbo, Yoruba and Hausa beyond greetings and simple phrases are weak (the reply LLM's limit, not the voice's). Read the limitations before using it for anything.
+
+## Components and licences
+
+| stage | component | licence |
+|---|---|---|
+| voice activity / turn-taking | [Silero VAD v5](https://github.com/snakers4/silero-vad) | MIT |
+| speech recognition | **[Dynamic ASR](https://huggingface.co/Modularcomputing/dynamic-asr)** `real` (Meta `omniASR_CTC_1B_v2` fine-tuned on licensed Nigerian speech; language-agnostic CTC + Latin-script mask). Fallback: stock [Meta Omnilingual ASR](https://github.com/facebookresearch/omnilingual-asr) `omniASR_CTC_1B` | weights CC-BY-SA-4.0 (base Apache-2.0) / Apache-2.0 |
+| language ID / mix | Dynamic Accent `tagger.py` (rules) + [GlotLID v3](https://huggingface.co/cis-lmu/glotlid) sliding windows | Apache-2.0 / Apache-2.0 |
+| reply LLM | [Qwen/Qwen3-30B-A3B-Instruct-2507-FP8](https://huggingface.co/Qwen/Qwen3-30B-A3B-Instruct-2507-FP8) served with [vLLM](https://github.com/vllm-project/vllm) | Apache-2.0 / Apache-2.0 |
+| text-to-speech | Dynamic Accent **Flash-DPO** ([Modularcomputing/dynamic-accent-pro-v1.1](https://huggingface.co/Modularcomputing/dynamic-accent-pro-v1.1), `checkpoints/flash`) on [Fun-CosyVoice3-0.5B-2512](https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512) | weights CC-BY-SA-4.0 (base Apache-2.0) |
+| diacritic restoration (only when a Yoruba/Igbo span has no tone marks) | Dynamic Accent ByT5 restorer (base `google/byt5-small`) | CC-BY-SA-4.0 (base Apache-2.0) |
+| this repo's code | orchestration, mix controller, validator, servers | **Apache-2.0** |
+
+Evaluation-only models (not part of the agent): Whisper large-v3-turbo (MIT), omniASR CTC-300M / LLM-1B / LLM-3B (Apache-2.0), Mistral-Small-3.2-24B-Instruct-2506 (Apache-2.0), MADLAD-400-3B-MT (Apache-2.0, back-translation).
+Evaluation data: WAXAL test splits (Media Trust ibo/yor/hau/pcm subsets, CC-BY-SA-4.0), FLEURS validation (CC-BY-4.0).
+Not used, on licence grounds: African Voices / WaZoBia data, NaijaVoices (non-commercial), N-ATLaS (1,000-user cap, share-alike). Gemma 3 was not tested (gated).
+
+This repo ships **code and config only**; each model is downloaded from its own repository under its own licence. Audio samples are made with the CC-BY-SA-4.0 TTS and are CC-BY-SA-4.0.
+
+## How the language mix works
+
+1. **Per-word language ID.** The release tagger labels every word (`eng/pcm/ibo/yor/hau`) from diacritics, lexicons and Pidgin markers; ASR text has no punctuation or tone marks, so GlotLID scores 4-word sliding windows and confident windows override the rule labels. A sentence-level pass turns English-looking sentences with Pidgin markers into Pidgin.
+2. **Mix estimate.** Word shares plus patterns: `greeting:<lang>`, `closing:<lang>`, `discourse:<lang>`, intra- vs inter-sentential switching, blended 30 % with the conversation so far. ASR-misspelt greetings ("nde wo", "sanno", "dalo") are matched against a variant list.
+3. **Constrained reply.** The LLM gets a natural-language target ("English about 80 %, Igbo about 20 %; open with a short Igbo greeting"), a phrasebook of correct greetings/thanks/closings, four examples, and must tag every span (`<eng>…</eng> <ibo>…</ibo>`), which is exactly the input format of the Dynamic Accent TTS.
+4. **Validator.** Each of 3 candidates is scored: tag format, total-variation distance between target and reply mix (declared tags averaged with re-measured LID), every target language ≥ 15 % present, greeting kept, 2–40 words, no loops, no echo of the user's words. The best passing candidate wins; otherwise one stricter round of 3 more; otherwise a fallback (best candidate, or for an Igbo/Yoruba/Hausa/Pidgin-led turn a correct phrasebook opener in that language plus a short clarification request).
+
+## Duplex behaviour
+
+- 32 ms Silero frames; speech start after 96 ms; end of turn after 600 ms of silence.
+- **Speculative start:** ASR → LLM → first TTS chunk begin at 256 ms of silence; audio is released at end of turn, and dropped if the user resumes.
+- **Barge-in:** 192 ms of user speech while the agent speaks (or has committed to speak) stops playback at once (client queue flushed); the interrupted reply is kept in history as what was actually played + `[interrupted]`.
+- **Backchannels:** during a long user turn (≥ 4.5 s), at a 288–448 ms pause, at most every 10 s: "Mm-hmm" (English) / "Ehen" (Pidgin, Igbo, Yoruba) / "To" (Hausa).
+- **Streaming TTS:** reply cut into a short first chunk (≤ 6–10 words, at a comma/greeting/conjunction), then sentences ≤ 22 words, each synthesised while the previous one plays.
+
+## Run it
+
+Tested on one H100 80 GB (vLLM 0.32, torch 2.x). Set up the Dynamic Accent TTS from `Modularcomputing/dynamic-accent-pro-v1.1` as its README describes, then:
+
+```bash
+pip install -r requirements.txt            # duplex server env (the TTS env)
+pip install "omnilingual-asr>=0.2.0" huggingface_hub fastapi uvicorn   # ASR worker env (can be a separate venv)
+# ASR choice: ASR_MODEL=dynamic-asr (default) | dynamic-asr-cs (experimental code-switch) | omni (stock omniASR_CTC_1B)
+pip install vllm                            # LLM server env
+DA_REL=/path/to/dynamic-accent-pro-v1.1 ./launch.sh   # -> http://localhost:7860 (use headphones)
+```
+
+Offline scripted conversations (same components, simulated real-time clock; latency = end-of-turn wait + measured compute):
+
+```bash
+cd code && python offline.py convo scenarios.json out/          # demo conversations -> wav + stereo (user L / agent R) + json log
+python offline.py eval list.jsonl out/ 40                        # batch latency / mix / barge-in eval on audio clips
+```
+
+## Evaluation
+
+All numbers from one H100 80 GB with the three processes sharing the GPU. Latency = end of the user's speech (VAD) to the first agent audio sample, from the offline driver: the end-of-turn wait is simulated frame by frame and every compute stage is timed on the real GPU (network and audio-device buffers excluded; the live WebSocket path measured 1.9-2.5 s over the public internet on two test turns). Held-out data only; whether Meta's ASR saw WAXAL/FLEURS in training is unknown.
+
+### ASR (229 held-out clips; WER / CER; CER without diacritics in brackets)
+
+| model | eng | pcm | ibo | yor | hau | cs | latency p50 / p90 per clip | CS mix TVD after ASR (rules / +GlotLID windows) |
+|---|---|---|---|---|---|---|---|---|
+| omniASR_CTC_1B | 0.14 / 0.07 (0.07) | 0.38 / 0.17 (0.17) | 0.63 / 0.20 (0.16) | 0.49 / 0.19 (0.12) | 0.30 / 0.09 (0.09) | 0.33 / 0.13 (0.10) | 44 / 50 ms | 0.402 / 0.277 |
+| omniASR_CTC_300M | 0.13 / 0.05 (0.05) | 0.58 / 0.23 (0.23) | 0.70 / 0.21 (0.17) | 0.59 / 0.21 (0.14) | 0.40 / 0.13 (0.13) | 0.40 / 0.14 (0.12) | 27 / 30 ms | 0.381 / 0.381 |
+| omniASR_LLM_1B | 0.07 / 0.03 (0.03) | 0.33 / 0.15 (0.15) | 0.54 / 0.17 (0.12) | 0.40 / 0.16 (0.11) | 0.30 / 0.09 (0.09) | 0.32 / 0.14 (0.11) | 1037 / 1802 ms | 0.4 / 0.4 |
+| omniASR_LLM_3B | 0.06 / 0.03 (0.03) | 0.43 / 0.18 (0.18) | 0.63 / 0.21 (0.15) | 0.35 / 0.14 (0.10) | 0.30 / 0.09 (0.09) | 0.32 / 0.15 (0.13) | 848 / 1760 ms | 0.352 / 0.352 |
+| whisper_turbo | 0.03 / 0.01 (0.01) | 0.42 / 0.21 (0.21) | 1.06 / 0.41 (0.34) | 1.05 / 0.52 (0.29) | 0.90 / 0.33 (0.32) | 0.81 / 0.62 (0.61) | 119 / 208 ms | 0.51 / 0.51 |
+| omniASR_CTC_1B_latin | 0.14 / 0.07 (0.07) | 0.38 / 0.17 (0.17) | 0.63 / 0.20 (0.16) | 0.49 / 0.19 (0.12) | 0.30 / 0.09 (0.09) | 0.33 / 0.13 (0.10) | 45 / 54 ms | 0.402 / 0.277 |
+| omniASR_CTC_1B_v2 (stock base) | 0.10 / 0.02 (0.02) | 0.34 / 0.12 (0.12) | 0.46 / 0.13 (0.11) | 0.40 / 0.13 (0.10) | 0.32 / 0.11 (0.11) | 0.29 / 0.10 (0.09) | same CTC-1B arch, not re-timed (≈ CTC_1B) | not measured |
+| **Dynamic ASR real** (default) | 0.10 / 0.03 (0.03) | 0.19 / 0.07 (0.07) | 0.32 / 0.09 (0.08) | 0.35 / 0.12 (0.08) | 0.25 / 0.08 (0.08) | 0.24 / 0.08 (0.06) | same CTC-1B arch, not re-timed (≈ CTC_1B) | not measured |
+| Dynamic ASR code-switch (experimental) | 0.09 / 0.02 (0.02) | 0.19 / 0.08 (0.08) | 0.32 / 0.10 (0.08) | 0.35 / 0.12 (0.08) | 0.25 / 0.08 (0.08) | 0.22 / 0.07 (0.06) | same CTC-1B arch, not re-timed (≈ CTC_1B) | not measured |
+
+**Note:** the end-to-end latency tables below (and the per-clip ASR latency column) were measured with `omniASR_CTC_1B` as the ASR, before Dynamic ASR existed. Dynamic ASR is the same size and architecture (1B wav2vec2 CTC, greedy decoding), so latency is expected to be similar but was not re-measured. Dynamic ASR rows use the same 229 clips and scorer; see the [Dynamic ASR card](https://huggingface.co/Modularcomputing/dynamic-asr) for the code-switch tests.
+
+LID on reference CS text (upper bound): word accuracy 0.9, mix TVD 0.133.
+
+### Reply LLM (28 prompts: 16 mixed/greeting/Pidgin/English written prompts + 12 native WAXAL sentences ibo/yor/hau)
+
+| model | subset | first candidate passes | regen rate | fallback | mean mix TVD | GlotLID agrees with span tag | judge 1-5 (back-translated) | LLM s (n=3) |
+|---|---|---|---|---|---|---|---|---|
+| mistral-small-3.2-24b | mixed | 0.93 | 0.07 | 0.0 | 0.027 | 1.0 | 4 | 0.433 |
+| mistral-small-3.2-24b | native | 0.92 | 0.08 | 0.0 | 0.008 | 1.0 | 1 | 0.584 |
+| mistral-small-3.2-24b | all | 0.89 | 0.07 | 0.0 | 0.017 | 1.0 | 2.79 | 0.497 |
+| qwen3-30b-a3b-2507 | mixed | 0.86 | 0.07 | 0.0 | 0.018 | 1.0 | 4.43 | 0.277 |
+| qwen3-30b-a3b-2507 | native | 0.92 | 0.0 | 0.0 | 0.029 | 1.0 | 1.58 | 0.41 |
+| qwen3-30b-a3b-2507 | all | 0.89 | 0.04 | 0.0 | 0.021 | 1.0 | 3.25 | 0.317 |
+
+### Latency, batch eval (80 turns; seconds, p50 / p90)
+
+| stage | p50 | p90 |
+|---|---|---|
+| end-of-turn wait (VAD silence) | 0.576 | 0.576 |
+| ASR final | 0.036 | 0.042 |
+| LID + mix | 0.013 | 0.03 |
+| LLM + validator rounds | 0.389 | 0.791 |
+| TTS first chunk | 0.863 | 1.69 |
+| compute total (ASR->first chunk) | 1.381 | 2.211 |
+| **end of user speech -> first agent audio** | 1.637 | 2.467 |
+
+Mix adherence (batch_eval): mean TVD(target, reply) 0.124 (p90 0.451); first candidate passes 0.588; regenerated 0.037; fallback 0.15 (safe phrasebook 0.113); 39/80 turns had a multi-language target.
+
+### Latency, demos (19 turns; seconds, p50 / p90)
+
+| stage | p50 | p90 |
+|---|---|---|
+| end-of-turn wait (VAD silence) | 0.576 | 0.576 |
+| ASR final | 0.057 | 0.069 |
+| LID + mix | 0.005 | 0.016 |
+| LLM + validator rounds | 0.246 | 0.537 |
+| TTS first chunk | 0.795 | 1.168 |
+| compute total (ASR->first chunk) | 1.139 | 1.488 |
+| **end of user speech -> first agent audio** | 1.395 | 1.744 |
+
+Mix adherence (demos): mean TVD(target, reply) 0.084 (p90 0.162); first candidate passes 0.789; regenerated 0.0; fallback 0.053 (safe phrasebook 0.053); 11/19 turns had a multi-language target.
+
+Barge-in (batch eval, n=40): user speech onset -> agent audio stopped p50 0.235 s, p90 0.3 s (VAD detection p50 0.195 s + 40 ms output flush).
+
+
+## Demos
+
+`samples/` holds four scripted conversations made with the offline driver (agent: Flash-DPO male anchor). **User turns in these samples are TTS-synthesised** (Flash female voices), not real people. Each has a JSON log with the ASR text, the target mix, the reply, the validator outcome and per-stage latency.
+
+## Limitations (read before use)
+
+- **Igbo, Yoruba and Hausa content is weak beyond greetings, thanks and short phrases.** The reply LLM (and every permissive LLM we tried) parrots or garbles longer native-language replies; a back-translation judge scored native-language replies 1.6/5 versus 4.4/5 for mixed/greeting/Pidgin prompts. The validator catches echoing and drift and falls back to an honest short reply, which means native-led turns often get "Sannu! Sorry, I did not catch all of that…".
+- ASR on accented or synthetic speech misspells Igbo/Yoruba greetings (and loses tone marks), which can hide the user's language from the mix estimate; a variant list covers common greetings only.
+- Pidgin vs English is the hardest LID boundary; Pidgin detection uses marker words.
+- Latency p90 is driven by the TTS first chunk (autoregressive 0.5B LM, ~17 ms per audio token, ~0.4-0.6 x real time on a shared H100).
+- Barge-in uses VAD only: without headphones or echo cancellation the agent's own voice can interrupt it. Backchannels are rule-based and can be too frequent on long read speech.
+- No safety filtering beyond the LLM's own; replies can be wrong. Research demo, not a product.
+
+## Citation / credits
+
+Built by Modular on Meta Omnilingual ASR, Silero VAD, GlotLID, Qwen3, vLLM, CosyVoice3 and the Dynamic Accent TTS (trained on licensed data only: WAXAL, FLEURS, Open.Bible, NaijaS2ST, IgboAPI, IgboSynCorp, BibleTTS, OpenSLR).

code/asr_server.py

@@ -0,0 +1,95 @@
+"""Streaming-capable ASR worker (runs in ~/venv_omni): Meta Omnilingual ASR CTC (Apache-2.0), language-agnostic decoding,
+so code-switched speech needs no language hint. POST /asr  body = raw float32 mono 16 kHz  -> {"text", "ms"}.
+Partial hypotheses = re-decoding the growing buffer (CTC is cheap: ~10-40 ms per call on H100)."""
+import os, time, numpy as np, torch
+import soundfile  # noqa: F401  (loads the bundled libsndfile before fairseq2n looks for it)
+from fastapi import FastAPI, Request
+from omnilingual_asr.models.inference.pipeline import ASRInferencePipeline
+# ASR model selection (env):
+#   ASR_MODEL=dynamic-asr (default) -> Dynamic ASR real-speech model  (Modularcomputing/dynamic-asr, real/model.pt)
+#   ASR_MODEL=dynamic-asr-cs        -> Dynamic ASR experimental code-switch model (codeswitch-experimental/model.pt)
+#   ASR_MODEL=omni                  -> stock Meta omniASR (ASR_CARD, default omniASR_CTC_1B)
+#   ASR_CKPT=<local .pt>            -> any local fine-tuned state_dict on top of ASR_CARD
+# Dynamic ASR = fine-tuned omniASR_CTC_1B_v2 (same CTC architecture; needs omnilingual-asr>=0.2.0, or patch_v2.sh on 0.1.0).
+# If the Dynamic ASR weights cannot be loaded, the worker falls back to stock omniASR_CTC_1B.
+LC = {"ibo": "ibo_Latn", "yor": "yor_Latn", "hau": "hau_Latn", "pcm": "pcm_Latn", "eng": "eng_Latn"}
+import unicodedata, types
+import omnilingual_asr.models.inference.pipeline as PL
+DYN = {"dynamic-asr": "real/model.pt", "dynamic-asr-cs": "codeswitch-experimental/model.pt"}
+MODEL = os.environ.get("ASR_MODEL", "dynamic-asr")
+CKPT = os.environ.get("ASR_CKPT", "")
+CARD = os.environ.get("ASR_CARD", "omniASR_CTC_1B_v2" if (MODEL in DYN or CKPT) else "omniASR_CTC_1B")
+
+def _load():
+    global CARD, CKPT, MODEL
+    if MODEL in DYN:
+        CARD = "omniASR_CTC_1B_v2"          # Dynamic ASR is always on the v2 base
+    if MODEL in DYN and not CKPT:
+        try:
+            from huggingface_hub import hf_hub_download
+            CKPT = hf_hub_download(os.environ.get("ASR_REPO", "Modularcomputing/dynamic-asr"), DYN[MODEL])
+        except Exception as e:
+            print("Dynamic ASR download failed, falling back to omniASR_CTC_1B:", repr(e)[:200], flush=True)
+            CARD, MODEL, CKPT = "omniASR_CTC_1B", "omni", ""
+    try:
+        pipe = ASRInferencePipeline(model_card=CARD)
+        if CKPT:
+            ck = torch.load(CKPT, map_location="cpu", weights_only=False)
+            print("ASR_CKPT", CKPT, pipe.model.load_state_dict({k: v.to(pipe.dtype) for k, v in ck["model"].items()}), flush=True)
+            del ck
+        return pipe
+    except Exception as e:
+        if CARD == "omniASR_CTC_1B":
+            raise
+        print("Dynamic ASR load failed (", repr(e)[:200], ") -> falling back to omniASR_CTC_1B", flush=True)
+        CARD, MODEL, CKPT = "omniASR_CTC_1B", "omni", ""
+        return ASRInferencePipeline(model_card=CARD)
+
+p = _load()
+LATIN_ONLY = os.environ.get("ASR_LATIN_ONLY", "1") == "1"
+
+def _ok(sx):
+    for c in sx:
+        cat = unicodedata.category(c)
+        if cat[0] == "M":                                   # combining marks: only the generic combining diacritics block
+            if not ("\u0300" <= c <= "\u036f"):
+                return False
+        elif c.isalpha() and "LATIN" not in unicodedata.name(c, ""):
+            return False
+    return True
+
+_BAD = {}
+def _apply_latin(self, batch):
+    """CTC greedy decoding restricted to Latin-script tokens: all five target languages are written in Latin script, and the
+    language-agnostic CTC otherwise drifts into Arabic/other scripts on accented or synthetic speech."""
+    bl = PL.BatchLayout(batch.source_seqs.shape, seq_lens=batch.source_seq_lens, device=batch.source_seqs.device)
+    logits, bl_out = self.model(batch.source_seqs, bl)
+    V = logits.shape[-1]
+    if V not in _BAD:
+        bad = [i for i in range(V) if not _ok(self.token_decoder(torch.tensor([i])))]
+        _BAD[V] = torch.tensor(bad, dtype=torch.long, device=logits.device); print("latin mask: blocked", len(bad), "of", V, flush=True)
+    logits[..., _BAD[V]] = -float("inf")
+    pred = torch.argmax(logits, dim=-1); out = []
+    for i in range(pred.shape[0]):
+        seq = pred[i][: bl_out.seq_lens[i]]; m = torch.ones(seq.shape[0], dtype=torch.bool, device=seq.device); m[1:] = seq[1:] != seq[:-1]
+        out.append(self.token_decoder(seq[m]))
+    return out
+
+if LATIN_ONLY and "CTC" in CARD:
+    p._apply_model_wav2vec2asr = types.MethodType(_apply_latin, p)
+app = FastAPI()
+w = np.zeros(16000, np.float32); p.transcribe([{"waveform": w, "sample_rate": 16000}], batch_size=1)
+
+@app.post("/asr")
+async def asr(req: Request, lang: str = ""):
+    y = np.frombuffer(await req.body(), dtype=np.float32).copy()
+    if len(y) < 1600:
+        return {"text": "", "ms": 0.0}
+    t = time.perf_counter()
+    kw = {} if "CTC" in CARD or not lang else dict(lang=[LC.get(lang, "eng_Latn")])
+    h = p.transcribe([{"waveform": y, "sample_rate": 16000}], batch_size=1, **kw)[0]
+    return {"text": h, "ms": round(1000 * (time.perf_counter() - t), 1), "card": CARD}
+
+@app.get("/health")
+def health():
+    return {"ok": True, "model": MODEL, "card": CARD, "ckpt": CKPT}

code/eval/analyze.py

@@ -0,0 +1,29 @@
+"""Aggregate offline batch-eval + demo logs into results/eval_summary.json (latency per stage p50/p90, barge-in, mix adherence)."""
+import json, glob, sys, statistics as st, os
+base = sys.argv[1] if len(sys.argv) > 1 else "results/out1"
+demo_dir = sys.argv[2] if len(sys.argv) > 2 else f"{base}/demos"
+def q(x, p):
+    x = sorted(v for v in x if v is not None); return round(x[min(len(x) - 1, int(p * len(x)))], 3) if x else None
+def pq(x): return dict(p50=q(x, .5), p90=q(x, .9), n=len([v for v in x if v is not None]))
+E = json.load(open(f"{base}/eval/eval_turns.json"))
+turns = [t for e in E for t in e["turns"]]
+D = [t for f in sorted(glob.glob(f"{demo_dir}/*.json")) for t in json.load(open(f))["turns"]]
+def lat(T):
+    return {k: pq([t.get(k) for t in T]) for k in ("latency_s", "eot_wait_s", "asr_s", "mix_s", "llm_s", "tts_first_s", "compute_to_first_audio_s")}
+def mixstats(T):
+    n = len(T)
+    return dict(n=n, tvd_mean=round(st.mean(t["tvd"] for t in T if t["tvd"] is not None), 3), tvd_p90=q([t["tvd"] for t in T], .9),
+                first_candidate_ok=round(sum(t["first_cand_ok"] for t in T) / n, 3), regen_rate=round(sum(t["how"].startswith("regen") for t in T) / n, 3),
+                fallback_rate=round(sum(t["how"] in ("best_failed", "plain_matrix", "safe_phrasebook") for t in T) / n, 3), safe_fallback_rate=round(sum(t["how"] == "safe_phrasebook" for t in T) / n, 3),
+                multi_lang_targets=sum(len(t["target"]["mix"]) > 1 for t in T))
+by_lang = {}
+for e in E:
+    t = e["turns"][0]; by_lang.setdefault(e["lang"], []).append(t)
+barge = [t["barge"] for t in turns if t.get("barge")]
+out = dict(
+    batch_eval=dict(desc="40 real held-out clips (WAXAL test ibo/yor/hau/pcm, FLEURS val eng, concatenated code-switch), each user turn followed by a barge-in clip 1.0 s into the reply (that clip is answered too)",
+                    latency=lat(turns), mix=mixstats(turns), per_lang_first_turn={l: dict(lat=pq([t["latency_s"] for t in T]), **mixstats(T)) for l, T in by_lang.items()},
+                    barge_in=dict(n=len(barge), reaction_s=pq([b["reaction_s"] for b in barge]), detected_minus_onset=pq([b["detected"] - b["user_onset"] for b in barge])),
+                    backchannels=sum(len(t.get("backchannels", [])) for t in turns)),
+    demos=dict(n_turns=len(D), latency=lat(D), mix=mixstats(D), barge_in=[t["barge"] for t in D if t.get("barge")], backchannels=sum(len(t.get("backchannels", [])) for t in D)))
+os.makedirs("results", exist_ok=True); json.dump(out, open("results/eval_summary.json", "w"), indent=1); print(json.dumps(out, indent=1))

code/eval/asr_omni.py

@@ -0,0 +1,27 @@
+"""omniASR candidates on the eval set: transcripts + single-clip latency + CTC streaming (prefix re-decode) sim.
+~/venv_omni/bin/python asr_omni.py <card> -> /scratch/duplex/asr_eval/hyp_<card>.jsonl"""
+import json, sys, time, numpy as np, soundfile as sf, torch
+from omnilingual_asr.models.inference.pipeline import ASRInferencePipeline
+LC = {"ibo": "ibo_Latn", "yor": "yor_Latn", "hau": "hau_Latn", "pcm": "pcm_Latn", "eng": "eng_Latn"}
+card = sys.argv[1]; D = "/scratch/duplex/asr_eval"
+R = [json.loads(l) for l in open(f"{D}/list.jsonl")]
+p = ASRInferencePipeline(model_card=card)
+ctc = "CTC" in card
+def tr(w, lang):
+    kw = {} if ctc or lang is None else dict(lang=[LC[lang]])
+    return p.transcribe([{"waveform": w, "sample_rate": 16000}], batch_size=1, **kw)[0]
+w0 = sf.read(R[0]["wav"], dtype="float32")[0]
+for _ in range(3): tr(w0, "eng")
+out = open(f"{D}/hyp_{card}.jsonl", "w")
+for r in R:
+    w = sf.read(r["wav"], dtype="float32")[0]
+    lang = r["lang"] if r["lang"] != "cs" else None     # LLM cards: CS clips get no language hint
+    torch.cuda.synchronize(); t = time.perf_counter(); h = tr(w, lang); torch.cuda.synchronize(); dt = time.perf_counter() - t
+    rec = dict(id=r["id"], hyp=h, lat=round(dt, 4), dur=r["dur"])
+    if ctc and r["dur"] > 4 and int(r["id"][-3:]) % 4 == 0:     # streaming sim: re-decode growing prefix every 0.64 s
+        parts = []
+        for e in np.arange(0.64, r["dur"], 0.64):
+            t = time.perf_counter(); hp = tr(w[: int(e * 16000)], None); parts.append((round(float(e), 2), hp, round(time.perf_counter() - t, 4)))
+        rec["partials"] = parts
+    out.write(json.dumps(rec, ensure_ascii=False) + "\n"); out.flush()
+print("OMNI_DONE", card)

code/eval/asr_score.py

@@ -0,0 +1,66 @@
+"""Score ASR candidates: WER/CER per language (lowercased, punctuation stripped), diacritic-folded CER, latency, streaming
+partial stability, and how well the mix estimate survives ASR (TVD between mix(ref text) and mix(hyp)). ~/venv/bin/python asr_score.py"""
+import json, glob, re, unicodedata, statistics as st, os, sys
+import jiwer
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import mix as M
+D = "/scratch/duplex/asr_eval"
+R = {json.loads(l)["id"]: json.loads(l) for l in open(f"{D}/list.jsonl")}
+def norm(s):
+    s = unicodedata.normalize("NFC", s.lower()); s = re.sub(r"[^\w\s']", " ", s); return re.sub(r"\s+", " ", s).strip()
+def fold(s):
+    return "".join(c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn")
+def q(x, p):
+    x = sorted(x); return x[min(len(x) - 1, int(p * len(x)))] if x else None
+out = {}
+for f in sorted(glob.glob(f"{D}/hyp_*.jsonl")):
+    if len(sys.argv) > 1 and sys.argv[1] not in f: continue
+    tag = f.split("hyp_")[1][:-6]; H = [json.loads(l) for l in open(f)]
+    if len(H) < len(R) * 0.9: print("incomplete", tag, len(H)); continue
+    res = {}
+    for lg in ["eng", "pcm", "ibo", "yor", "hau", "cs"]:
+        S = [h for h in H if R[h["id"]]["lang"] == lg]
+        refs = [norm(R[h["id"]]["text"]) for h in S]; hyps = [norm(h["hyp"]) or "x" for h in S]
+        res[lg] = dict(n=len(S), wer=round(jiwer.wer(refs, hyps), 3), cer=round(jiwer.cer(refs, hyps), 3),
+                       cer_nodiac=round(jiwer.cer([fold(r) for r in refs], [fold(h) for h in hyps]), 3))
+    lat = [h["lat"] for h in H]; rtf = [h["lat"] / h["dur"] for h in H]
+    res["latency"] = dict(p50_s=round(q(lat, .5), 3), p90_s=round(q(lat, .9), 3), rtf_p50=round(q(rtf, .5), 4))
+    P = [h for h in H if h.get("partials")]
+    if P:   # stability: share of final words already present (in order prefix) in the partial 1.28 s before the end
+        stab, plat = [], []
+        for h in P:
+            fin = norm(h["hyp"]).split(); parts = h["partials"]
+            if len(parts) >= 2 and fin:
+                pw = norm(parts[-2][1]).split(); k = 0
+                while k < min(len(pw), len(fin)) and pw[k] == fin[k]: k += 1
+                stab.append(k / max(1, len(pw)))
+            plat += [p[2] for p in parts]
+        res["streaming"] = dict(n=len(P), partial_prefix_stability=round(st.mean(stab), 3) if stab else None, partial_call_p50_s=round(q(plat, .5), 4), partial_call_p90_s=round(q(plat, .9), 4))
+    # mix estimate after ASR (CS clips): TVD(mix(ref), mix(hyp)), rules-only and with GlotLID
+    tv = {}
+    for lid in ("none", "glotlid"):
+        v = []
+        for h in H:
+            r = R[h["id"]]
+            if r["lang"] != "cs": continue
+            ref_mix = {}
+            for l in r["wlang"]: ref_mix[l] = ref_mix.get(l, 0) + 1 / len(r["wlang"])
+            v.append(M.tvd(ref_mix, M.estimate(h["hyp"], lid=lid, asr=True)["mix"]))
+        tv[lid] = round(st.mean(v), 3) if v else None
+        mono = [M.estimate(h["hyp"], lid=lid, asr=True)["mix"].get(R[h["id"]]["lang"], 0) for h in H if R[h["id"]]["lang"] in ("ibo", "yor", "hau", "pcm")]
+        tv[lid + "_mono_true_share"] = round(st.mean(mono), 3)
+    res["cs_mix_tvd_after_asr"] = tv
+    out[tag] = res
+# LID on reference text (upper bound for the mix estimate)
+tv = {}
+for lid in ("none", "glotlid"):
+    v = []; acc = []
+    for r in R.values():
+        if r["lang"] != "cs": continue
+        ref_mix = {}
+        for l in r["wlang"]: ref_mix[l] = ref_mix.get(l, 0) + 1 / len(r["wlang"])
+        e = M.estimate(r["text"], lid=lid); v.append(M.tvd(ref_mix, e["mix"]))
+        if len(e["labels"]) == len(r["wlang"]): acc += [a == b for a, b in zip(e["labels"], r["wlang"])]
+    tv[lid] = dict(mix_tvd=round(st.mean(v), 3), word_acc=round(sum(acc) / len(acc), 3) if acc else None, n_words=len(acc))
+out["_lid_on_reference_text"] = tv
+json.dump(out, open(f"{D}/asr_scores.json", "w"), indent=1); print(json.dumps(out, indent=1))

code/eval/asr_via_server.py

@@ -0,0 +1,8 @@
+"""Re-run the ASR eval through the live ASR server (e.g. CTC_1B + Latin-script mask): python asr_via_server.py <tag>"""
+import json, sys, time, httpx, soundfile as sf, numpy as np
+D = "/scratch/duplex/asr_eval"; c = httpx.Client(timeout=30); out = open(f"{D}/hyp_{sys.argv[1]}.jsonl", "w")
+for l in open(f"{D}/list.jsonl"):
+    r = json.loads(l); y = sf.read(r["wav"], dtype="float32")[0]
+    t = time.perf_counter(); h = c.post("http://127.0.0.1:8002/asr", content=y.tobytes()).json(); dt = time.perf_counter() - t
+    out.write(json.dumps(dict(id=r["id"], hyp=h["text"], lat=round(dt, 4), dur=r["dur"]), ensure_ascii=False) + "\n")
+print("VIA_DONE")

code/eval/asr_whisper.py

@@ -0,0 +1,17 @@
+"""Whisper candidates (MIT) via transformers: ~/venv/bin/python asr_whisper.py <hf_id> <tag>"""
+import json, sys, time, torch, soundfile as sf
+from transformers import pipeline
+mid, tag = sys.argv[1], sys.argv[2]; D = "/scratch/duplex/asr_eval"
+WL = {"eng": "en", "yor": "yo", "hau": "ha", "pcm": "en", "ibo": None, "cs": None}
+asr = pipeline("automatic-speech-recognition", model=mid, torch_dtype=torch.float16, device="cuda")
+R = [json.loads(l) for l in open(f"{D}/list.jsonl")]
+out = open(f"{D}/hyp_{tag}.jsonl", "w")
+for i, r in enumerate(R):
+    w = sf.read(r["wav"], dtype="float32")[0]
+    gk = {"task": "transcribe"}
+    if WL[r["lang"]]: gk["language"] = WL[r["lang"]]
+    torch.cuda.synchronize(); t = time.perf_counter()
+    h = asr({"raw": w, "sampling_rate": 16000}, generate_kwargs=gk)["text"].strip()
+    torch.cuda.synchronize(); dt = time.perf_counter() - t
+    out.write(json.dumps(dict(id=r["id"], hyp=h, lat=round(dt, 4), dur=r["dur"]), ensure_ascii=False) + "\n"); out.flush()
+print("WHISPER_DONE", tag)

code/eval/build_asr_eval.py

@@ -0,0 +1,48 @@
+"""Build held-out ASR/LID eval set -> /scratch/duplex/asr_eval/{wav/*.wav, list.jsonl}
+Mono: WAXAL test (ibo/yor/hau/pcm, 40 each, spread over speakers), FLEURS val eng (30).
+CS: 40 synthetic code-switched clips = two held-out clips of different languages concatenated (0.25 s gap), ref = joined text + per-word lang labels.
+All splits are test/val/dev splits never used in TTS training; whether the ASR models saw them is unknown (noted in report)."""
+import json, os, random, numpy as np, soundfile as sf, librosa
+H = os.path.expanduser("~"); OUT = "/scratch/duplex/asr_eval"; os.makedirs(OUT + "/wav", exist_ok=True)
+random.seed(7)
+def rows(d, pred):
+    R = [json.loads(l) for l in open(f"{d}/meta.jsonl")]
+    return [r for r in R if pred(r)]
+def audio(d, r):
+    a = np.memmap(f"{d}/pcm24k.i16", dtype=np.int16, mode="r")
+    y = a[r["off24"]: r["off24"] + r["n24"]].astype(np.float32) / 32768
+    return librosa.resample(y, orig_sr=24000, target_sr=16000)
+def pick(R, n):
+    by = {}
+    for r in R: by.setdefault(r.get("spk"), []).append(r)
+    for v in by.values(): random.shuffle(v)
+    out = []
+    while len(out) < n and any(by.values()):
+        for k in list(by):
+            if by[k] and len(out) < n: out.append(by[k].pop())
+    return out
+items = []
+W = f"{H}/data3/waxal"
+for L in ["ibo", "yor", "hau", "pcm"]:
+    for r in pick(rows(W, lambda r: r["lang"] == L and r["split"] == "test" and 1.5 < r["dur"] < 14), 40):
+        items.append(("waxal_test", W, r, L))
+F = f"{H}/data3/fleurs"
+for r in pick(rows(F, lambda r: r["lang"] == "eng" and r["split"] == "val" and r["dur"] < 16), 30): items.append(("fleurs_val", F, r, "eng"))
+recs = []; pool = {}
+for src, d, r, L in items:
+    y = audio(d, r); i = f"{L}_{len(recs):03d}"
+    sf.write(f"{OUT}/wav/{i}.wav", y, 16000)
+    rec = dict(id=i, wav=f"{OUT}/wav/{i}.wav", lang=L, text=r["text"], src=f"{src}:{r['id']}", dur=round(len(y) / 16000, 2),
+               wlang=[L] * len(r["text"].split()))
+    recs.append(rec); pool.setdefault(L, []).append((rec, y))
+pairs = [("eng", "ibo"), ("ibo", "eng"), ("eng", "yor"), ("yor", "eng"), ("eng", "hau"), ("hau", "eng"), ("pcm", "eng"), ("eng", "pcm"), ("pcm", "ibo"), ("pcm", "yor")]
+for k in range(40):
+    a, b = pairs[k % len(pairs)]
+    (ra, ya), (rb, yb) = random.choice(pool[a]), random.choice(pool[b])
+    y = np.concatenate([ya, np.zeros(4000, np.float32), yb]); i = f"cs_{k:03d}"
+    sf.write(f"{OUT}/wav/{i}.wav", y, 16000)
+    recs.append(dict(id=i, wav=f"{OUT}/wav/{i}.wav", lang="cs", langs=[a, b], text=ra["text"] + " " + rb["text"], src=f"concat:{ra['id']}+{rb['id']}",
+                     dur=round(len(y) / 16000, 2), wlang=ra["wlang"] + rb["wlang"]))
+with open(f"{OUT}/list.jsonl", "w") as f:
+    for r in recs: f.write(json.dumps(r, ensure_ascii=False) + "\n")
+from collections import Counter; print(Counter(r["lang"] for r in recs)); print("BUILD_DONE", len(recs))

code/eval/llm_eval.py

@@ -0,0 +1,84 @@
+"""Compare response LLMs on mix-matched replies.
+  gen <tag> prompts.json out.json   (needs a vLLM server at LLM_BASE; full Responder path: mix -> n=3 candidates -> validator -> regen)
+  bt  out1.json [out2.json ...]     (MADLAD-400-3B (Apache-2.0) back-translates the non-English spans of user + reply into English)
+  judge out1.json ...               (server LLM rates coherence/relevance of the back-translated reply, 1-5)
+  summary out1.json ..."""
+import sys, json, re, os, statistics as st
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+TAG = re.compile(r"<(eng|ibo|yor|hau|pcm)>(.*?)</\1>", re.S)
+
+def add_native(P):
+    """+4 native WAXAL test transcripts per ibo/yor/hau (monolingual user turns)"""
+    L = [json.loads(l) for l in open("/scratch/duplex/asr_eval/list.jsonl")]
+    for lg in ("ibo", "yor", "hau"):
+        rs = [r for r in L if r["lang"] == lg and len(r["text"].split()) >= 6][:4]
+        P += [dict(id=f"n_{lg}_{i}", kind=f"{lg} native", text=r["text"]) for i, r in enumerate(rs)]
+    return P
+
+def gen(tag, pf, out):
+    from respond import Responder
+    import mix as M
+    R = Responder(); res = []
+    for p in add_native(json.load(open(pf))):
+        R.style = None
+        r = R.reply(p["text"])
+        spans = TAG.findall(r["reply"]); glot = []
+        for l, t in spans:
+            if l in ("ibo", "yor", "hau") and len(t.split()) >= 2:
+                s = M.tagger("glotlid")._lid_scores(t); glot.append(max(s, key=s.get) == l)
+        res.append(dict(p, model=tag, **{k: r[k] for k in ("target", "reply", "reply_mix", "tvd", "how", "first_cand_ok", "any_first_round_ok", "t_llm", "rounds")}, glot_ok=glot))
+        print(json.dumps({"id": p["id"], "reply": r["reply"], "how": r["how"], "tvd": r["tvd"]}, ensure_ascii=False), flush=True)
+    json.dump(res, open(out, "w"), ensure_ascii=False, indent=1)
+
+def bt(files):
+    import torch
+    from transformers import T5ForConditionalGeneration, T5Tokenizer
+    import mix as M
+    tok = T5Tokenizer.from_pretrained("google/madlad400-3b-mt"); m = T5ForConditionalGeneration.from_pretrained("google/madlad400-3b-mt", torch_dtype=torch.bfloat16).cuda().eval()
+    def tr(s):
+        ids = tok("<2en> " + s, return_tensors="pt").input_ids.cuda()
+        with torch.no_grad(): o = m.generate(ids, max_new_tokens=3 * len(s.split()) + 16, num_beams=2, no_repeat_ngram_size=4)
+        return tok.decode(o[0], skip_special_tokens=True)
+    def en(tagged):
+        return " ".join(t if l in ("eng", "pcm") else tr(t) for l, t in TAG.findall(tagged))
+    for f in files:
+        R = json.load(open(f))
+        for r in R:
+            r["user_bt"] = en(M.tagger("glotlid").tag(r["text"])); r["reply_bt"] = en(r["reply"])
+        json.dump(R, open(f, "w"), ensure_ascii=False, indent=1); print("bt", f, flush=True)
+
+def judge(files):
+    from llm import LLM
+    J = LLM()
+    for f in files:
+        R = json.load(open(f))
+        for r in R:
+            q = (f"A user said (English gloss): \"{r['user_bt']}\"\nThe assistant replied (English back-translation): \"{r['reply_bt']}\"\n"
+                 "Rate from 1 to 5 how coherent, sensible and relevant the reply is as a spoken assistant response (5 = fully sensible and on-topic, "
+                 "1 = nonsense or off-topic). Output only the digit.")
+            o, _ = J.generate([{"role": "user", "content": q}], n=1, temperature=0.0, max_tokens=4)
+            m = re.search(r"[1-5]", o[0]); r["judge"] = int(m.group()) if m else None
+        json.dump(R, open(f, "w"), ensure_ascii=False, indent=1); print("judged", f, flush=True)
+
+def summary(files):
+    rows = []
+    for f in files:
+        R = json.load(open(f)); tag = R[0]["model"]
+        def grp(pred):
+            S = [r for r in R if pred(r)]
+            if not S: return None
+            g = [x for r in S for x in r["glot_ok"]]
+            j = [r["judge"] for r in S if r.get("judge")]
+            return dict(n=len(S), first_cand_ok=round(sum(r["first_cand_ok"] for r in S) / len(S), 2), regen=round(sum(r["how"] != "first_round" for r in S) / len(S), 2),
+                        fallback=round(sum(r["how"] in ("best_failed", "plain_matrix") for r in S) / len(S), 2), tvd=round(st.mean(r["tvd"] or 0 for r in S), 3),
+                        glot_ok=round(sum(g) / len(g), 2) if g else None, judge=round(st.mean(j), 2) if j else None, llm_s=round(st.median(r["t_llm"] for r in S), 3))
+        rows.append(dict(model=tag, all=grp(lambda r: True), mixed=grp(lambda r: not r["id"].startswith("n_") and not r["id"].startswith("e_")),
+                         native=grp(lambda r: r["id"].startswith("n_"))))
+    print(json.dumps(rows, indent=1)); return rows
+
+if __name__ == "__main__":
+    c = sys.argv[1]
+    if c == "gen": gen(sys.argv[2], sys.argv[3], sys.argv[4])
+    elif c == "bt": bt(sys.argv[2:])
+    elif c == "judge": judge(sys.argv[2:])
+    else: json.dump(summary(sys.argv[2:]), open("/scratch/duplex/llm_summary.json", "w"), indent=1)

code/eval/llm_prompts.json

@@ -0,0 +1,18 @@
+[
+ {"id":"g_ibo_1","kind":"eng+ibo greeting","text":"Ndewo! Good morning. Please, can you remind me what time the market closes today?"},
+ {"id":"g_ibo_2","kind":"eng+ibo greeting/closing","text":"Kedu? I am looking for a good place to buy fresh fish in Owerri. Daalụ."},
+ {"id":"g_yor_1","kind":"eng+yor greeting","text":"Ẹ káàárọ̀. I want to know how to get to the airport from Ikeja."},
+ {"id":"g_yor_2","kind":"eng+yor greeting","text":"Ẹ kú ìrọ̀lẹ́. My phone battery drains very fast, what should I do?"},
+ {"id":"g_hau_1","kind":"eng+hau greeting","text":"Sannu. Please help me plan a small birthday party for my daughter."},
+ {"id":"g_hau_2","kind":"eng+hau greeting/closing","text":"Barka da yamma. What is a good dinner I can cook in thirty minutes? Na gode."},
+ {"id":"g_pcm_1","kind":"eng+pcm opener","text":"How far? I need to write an email to my landlord about the leaking roof."},
+ {"id":"i_eng_ibo","kind":"intra eng/ibo","text":"Biko, how do I reset my bank app password? I forgot it."},
+ {"id":"i_eng_yor","kind":"intra eng/yor","text":"Jọ̀ọ́, can you check the weather for Lagos today?"},
+ {"id":"i_eng_hau","kind":"intra eng/hau","text":"Don Allah, how much is a bus ticket from Kano to Abuja?"},
+ {"id":"p_1","kind":"pcm","text":"Abeg, I don try call my brother but network no gree. Wetin I fit do?"},
+ {"id":"p_2","kind":"pcm","text":"My guy, this traffic no be here o. I go late for work today."},
+ {"id":"p_3","kind":"pcm","text":"Wetin be the best way to save money every month? Salary no dey last."},
+ {"id":"p_4","kind":"pcm+eng","text":"I wan start small business, maybe selling drinks. You think say e go work?"},
+ {"id":"e_1","kind":"eng","text":"Can you recommend a good book to read this weekend?"},
+ {"id":"e_2","kind":"eng","text":"What should I pack for a three day trip to Calabar?"}
+]

code/eval/make_report.py

@@ -0,0 +1,28 @@
+"""Fill REPORT.md tables from results/*.json. python make_report.py <eval_summary.json> <asr_scores.json> <llm_summary.json> -> prints markdown blocks"""
+import json, sys
+ev, asr, llm = (json.load(open(f)) for f in sys.argv[1:4])
+L = ["eng", "pcm", "ibo", "yor", "hau", "cs"]
+o = ["### ASR (229 held-out clips; WER / CER; CER without diacritics in brackets)", "",
+     "| model | " + " | ".join(L) + " | latency p50 / p90 per clip | CS mix TVD after ASR (rules / +GlotLID windows) |", "|---" * (len(L) + 3) + "|"]
+for k, v in asr.items():
+    if k.startswith("_"): continue
+    o.append(f"| {k} | " + " | ".join(f"{v[l]['wer']:.2f} / {v[l]['cer']:.2f} ({v[l]['cer_nodiac']:.2f})" for l in L) +
+             f" | {v['latency']['p50_s']*1000:.0f} / {v['latency']['p90_s']*1000:.0f} ms | {v['cs_mix_tvd_after_asr'].get('none')} / {v['cs_mix_tvd_after_asr'].get('glotlid')} |")
+o += ["", f"LID on reference CS text (upper bound): word accuracy {asr['_lid_on_reference_text']['glotlid']['word_acc']}, mix TVD {asr['_lid_on_reference_text']['glotlid']['mix_tvd']}.", ""]
+o += ["### Reply LLM (28 prompts: 16 mixed/greeting/Pidgin/English written prompts + 12 native WAXAL sentences ibo/yor/hau)", "",
+      "| model | subset | first candidate passes | regen rate | fallback | mean mix TVD | GlotLID agrees with span tag | judge 1-5 (back-translated) | LLM s (n=3) |", "|---|---|---|---|---|---|---|---|---|"]
+for r in llm:
+    for sub in ("mixed", "native", "all"):
+        g = r[sub]
+        o.append(f"| {r['model']} | {sub} | {g['first_cand_ok']} | {g['regen']} | {g['fallback']} | {g['tvd']} | {g['glot_ok']} | {g['judge']} | {g['llm_s']} |")
+for name in ("batch_eval", "demos"):
+    e = ev[name]; la = e["latency"]
+    o += ["", f"### Latency, {name.replace('_', ' ')} ({la['latency_s']['n']} turns; seconds, p50 / p90)", "", "| stage | p50 | p90 |", "|---|---|---|"]
+    for k, lab in [("eot_wait_s", "end-of-turn wait (VAD silence)"), ("asr_s", "ASR final"), ("mix_s", "LID + mix"), ("llm_s", "LLM + validator rounds"), ("tts_first_s", "TTS first chunk"),
+                   ("compute_to_first_audio_s", "compute total (ASR->first chunk)"), ("latency_s", "**end of user speech -> first agent audio**")]:
+        o.append(f"| {lab} | {la[k]['p50']} | {la[k]['p90']} |")
+    m = e["mix"]
+    o += ["", f"Mix adherence ({name}): mean TVD(target, reply) {m['tvd_mean']} (p90 {m['tvd_p90']}); first candidate passes {m['first_candidate_ok']}; regenerated {m['regen_rate']}; fallback {m['fallback_rate']}" + (f" (safe phrasebook {m.get('safe_fallback_rate')})" if 'safe_fallback_rate' in m else "") + f"; {m['multi_lang_targets']}/{m['n']} turns had a multi-language target."]
+b = ev["batch_eval"]["barge_in"]
+o += ["", f"Barge-in (batch eval, n={b['n']}): user speech onset -> agent audio stopped p50 {b['reaction_s']['p50']} s, p90 {b['reaction_s']['p90']} s (VAD detection p50 {b['detected_minus_onset']['p50']} s + 40 ms output flush)."]
+print("\n".join(o))

code/eval/ws_test.py

@@ -0,0 +1,34 @@
+"""Live-path test of server.py over a real WebSocket (real-time paced 32 ms frames).
+python ws_test.py <ws_url> <user1.wav> [<barge.wav>]  -> prints events; saves ws_test_agent.wav"""
+import asyncio, json, sys, time, numpy as np, soundfile as sf, librosa, websockets
+async def main(url, w1, w2=None):
+    def load(p):
+        y, sr = sf.read(p, dtype="float32"); y = y.mean(1) if y.ndim > 1 else y
+        return (librosa.resample(y, orig_sr=sr, target_sr=16000) * 32767).astype(np.int16)
+    u1 = load(w1); u2 = load(w2) if w2 else None
+    t0 = time.time(); ev = []; audio = []; first_audio = [None]; got_reply = [False]; got_reply = [False]
+    async with websockets.connect(url, max_size=2 ** 24) as ws:
+        async def rx():
+            async for m in ws:
+                t = round(time.time() - t0, 3)
+                if isinstance(m, bytes):
+                    audio.append(np.frombuffer(m, np.int16)); first_audio[0] = first_audio[0] or (t if got_reply[0] else None); ev.append((t, "audio", len(m) // 2 / 24000))
+                else:
+                    d = json.loads(m); got_reply[0] = got_reply[0] or d.get("type") == "reply"; ev.append((t, d.get("type"), {k: v for k, v in d.items() if k != "type"})); print(t, d, flush=True)
+        task = asyncio.create_task(rx())
+        async def send(y):
+            for i in range(0, len(y) - 511, 512):
+                await ws.send(y[i:i + 512].tobytes()); await asyncio.sleep(0.032)
+        z = np.zeros(512, np.int16)
+        await send(u1); print(round(time.time() - t0, 3), "user1 sent", flush=True)
+        for _ in range(int(6.0 / 0.032)):
+            await ws.send(z.tobytes()); await asyncio.sleep(0.032)
+            if u2 is not None and first_audio[0] and time.time() - t0 > first_audio[0] + 1.0: break
+        if u2 is not None:
+            print(round(time.time() - t0, 3), "barge-in user2 starts", flush=True); await send(u2)
+        for _ in range(int(8 / 0.032)):
+            await ws.send(z.tobytes()); await asyncio.sleep(0.032)
+        task.cancel()
+    if audio: sf.write("ws_test_agent.wav", np.concatenate(audio).astype(np.float32) / 32768, 24000)
+    print("WS_TEST_DONE", "first_audio_at", first_audio[0])
+asyncio.run(main(*sys.argv[1:]))

code/llm.py

@@ -0,0 +1,62 @@
+"""Response LLM: a reply that mirrors the user's language mix, with inline language tags for the Flash TTS.
+Talks to an OpenAI-compatible vLLM server (default http://127.0.0.1:8001/v1, served name "llm")."""
+import os, time, json, httpx
+
+PHRASEBOOK = """Phrasebook (use these exact spellings when you need them):
+Igbo: Ndewo (hello), Kedu? (how are you?), Ọ dị mma (it is fine/good), Ụtụtụ ọma (good morning), Ehihie ọma (good afternoon), Mgbede ọma (good evening), Nnọọ (welcome), Daalụ (thank you), Biko (please), Ee (yes), Mba (no), Ka ọ dị (goodbye), Chineke gozie gị (God bless you)
+Yoruba: Ẹ káàárọ̀ (good morning), Ẹ káàsán (good afternoon), Ẹ kú ìrọ̀lẹ́ (good evening), Ṣé dáadáa ni? (are you well?), Ẹ kú àbọ̀ (welcome), Ẹ ṣé / Ẹ ṣéun (thank you), Jọ̀ọ́ (please), Bẹ́ẹ̀ ni (yes), Rárá (no), Ó dàbọ̀ (goodbye), Ẹ pẹ̀lẹ́ (sorry/hello)
+Hausa: Sannu (hello), Barka da safiya (good morning), Barka da yamma (good evening), Ina kwana? (good morning, how was the night?), Lafiya lau (fine), Maraba (welcome), Na gode (thank you), Don Allah (please), Eh (yes), A'a (no), Sai an jima (see you later)
+Nigerian Pidgin: How far? (hi), I dey (I'm fine), Abeg (please), Wetin (what), No wahala (no problem), Na so (that's right), Thank you well well, We go see later (goodbye)"""
+
+SYSTEM = """You are Ada, a warm, quick Nigerian voice assistant in a live spoken conversation. Your words are spoken aloud by a code-switching text-to-speech voice.
+Language rule (most important): reply in the SAME language mix as the user, as described in the [Language target] note. If the user speaks mostly English with an Igbo greeting, you reply mostly in English with an Igbo greeting. If the user speaks Pidgin, reply in Pidgin. If the user speaks Yoruba, Hausa or Igbo, reply mainly in that language. Never add a language the user did not use. You are the assistant: answer and help; never repeat the user's sentence as if it were your own.
+Output format: wrap every stretch of words in a language tag: <eng>...</eng>, <pcm>...</pcm>, <ibo>...</ibo>, <yor>...</yor>, <hau>...</hau>. Nothing outside tags. One line.
+Style: spoken, natural, friendly. Start with a short opener of one to three words followed by a comma or full stop (for example 'Okay,' 'Ehen,' 'No wahala!' 'Sure,' or a greeting), then one or two short sentences, at most 35 words in total. No lists, no emojis, no markdown. Write numbers as words. Ask a short follow-up question when it helps.
+The user text comes from speech recognition: it has no punctuation and may misspell Igbo, Yoruba or Hausa words (for example "aca arow" for "Ẹ káàárọ̀", "baka da yamma" for "Barka da yamma", "dalo" for "Daalụ"). Understand what was meant, greet back in that language when the user greeted in it, and never copy misspelled words into your reply.
+Write Igbo and Yoruba with their proper letters and tone marks when you can. Keep Igbo, Yoruba and Hausa simple and correct; prefer common phrases over long complex sentences.
+""" + PHRASEBOOK
+
+SHOTS = [
+    ("Ndewo, good afternoon. Please, what time does the bank open on Saturday?",
+     "Match the user's language mix: English about 80%, Igbo about 20% of the words. The user greeted in Igbo: open with a short Igbo greeting.",
+     "<ibo>Ndewo!</ibo> <eng>On Saturdays most banks open at ten and close by two. Do you want me to check a particular branch?</eng>"),
+    ("How far? Abeg, wetin I fit cook tonight wey no go take long?",
+     "Match the user's language mix: Nigerian Pidgin about 100% of the words. Reply only in Nigerian Pidgin.",
+     "<pcm>I dey o! You fit make jollof spaghetti with egg, e no go pass thirty minutes. You get pepper for house?</pcm>"),
+    ("Ẹ káàárọ̀. I want to book a doctor's appointment for tomorrow.",
+     "Match the user's language mix: English about 80%, Yoruba about 20% of the words. The user greeted in Yoruba: open with a short Yoruba greeting.",
+     "<yor>Ẹ káàárọ̀!</yor> <eng>Sure, I can help with that. What time works best for you tomorrow?</eng>"),
+    ("Sannu. Ina so in sayi sabuwar waya.",
+     "Match the user's language mix: Hausa about 100% of the words. Reply only in Hausa.",
+     "<hau>Sannu da zuwa! Akwai wayoyi masu kyau da yawa. Nawa ne kasafin kuɗinka?</hau>"),
+]
+
+def build_messages(user_text, target_desc, history=(), shots=True):
+    m = [{"role": "system", "content": SYSTEM}]
+    if shots:
+        for u, d, a in SHOTS:
+            m += [{"role": "user", "content": f"{u}\n[Language target] {d}"}, {"role": "assistant", "content": a}]
+    for u, a in list(history)[-6:]:
+        m += [{"role": "user", "content": u}, {"role": "assistant", "content": a}]
+    m.append({"role": "user", "content": f"{user_text}\n[Language target] {target_desc}"})
+    return m
+
+class LLM:
+    def __init__(self, base=os.environ.get("LLM_BASE", "http://127.0.0.1:8001/v1"), model=os.environ.get("LLM_NAME", "llm"), thinking_kw=True):
+        self.base, self.model = base, model
+        self.cli = httpx.Client(timeout=60)
+        self.thinking_kw = thinking_kw
+
+    def generate(self, messages, n=3, temperature=0.7, max_tokens=110, top_p=0.9):
+        body = dict(model=self.model, messages=messages, n=n, temperature=temperature, top_p=top_p, max_tokens=max_tokens)
+        if self.thinking_kw:
+            body["chat_template_kwargs"] = {"enable_thinking": False}
+        t = time.perf_counter()
+        r = self.cli.post(self.base + "/chat/completions", json=body)
+        if r.status_code == 400 and self.thinking_kw:          # template without the kwarg
+            self.thinking_kw = False; return self.generate(messages, n, temperature, max_tokens, top_p)
+        r.raise_for_status()
+        outs = [c["message"]["content"] or "" for c in r.json()["choices"]]
+        import re
+        outs = [re.sub(r"<think>.*?</think>", "", o, flags=re.S).strip() for o in outs]
+        return outs, time.perf_counter() - t

code/mix.py

@@ -0,0 +1,136 @@
+"""Per-word language ID + language-mix estimate for a user turn (and for a reply).
+Reuses the Dynamic Accent release tagger (rule-based lexicons/diacritics/Pidgin markers, optional GlotLID clause scores).
+estimate("Ndewo, how are you doing today?") ->
+  {"mix": {"eng": 0.67, "ibo": 0.33}, "matrix": "eng", "pattern": ["greeting:ibo"], "words": [...], "labels": [...]}"""
+import re, unicodedata
+from tagger import Tagger, fold
+
+LANGS = ["eng", "pcm", "ibo", "yor", "hau"]
+NAMES = {"eng": "English", "pcm": "Nigerian Pidgin", "ibo": "Igbo", "yor": "Yoruba", "hau": "Hausa"}
+# greeting / closing / discourse lexicon (diacritics folded) -> language
+GREET = {"ibo": ["nde wo", "ndewoo", "ndeewo", "kedu", "nnoo", "ututu oma", "ehihie oma", "mgbede oma", "ka chi fo", "kedu ka i mere", "i bola chi"],
+         "yor": ["e kaaro", "e kaasan", "e kurole", "e ku irole", "bawo", "se daadaa ni", "e nle", "pele", "e kaabo", "e ku abo", "ekaaro", "ekaasan", "ekaro", "e karo", "e kaaro o"],
+         "hau": ["sannu", "sanu", "sanno", "baka da yamma", "baka da safiya", "barka", "barka da safiya", "barka da yamma", "ina kwana", "ina wuni", "yaya dai", "lafiya lau", "maraba"],
+         "pcm": ["how far", "how body", "wetin dey", "how you dey", "how una dey", "i hail"]}
+CLOSE = {"ibo": ["daalu", "dalu", "dalo", "daalo", "dalow", "ka o di", "ka emesia", "chukwu gozie"], "yor": ["e se", "o se", "o dabo", "e seun", "ese", "odabo"],
+         "hau": ["na gode", "sai an jima", "sai anjima", "nagode", "mun gode", "na godi", "nagodi"], "pcm": ["we go see", "thank you well well", "make we yarn later"]}
+DISC = {"pcm": ["abeg", "abi", "sha", "o", "na so", "omo", "wahala"], "yor": ["sha", "o", "ehen", "abi"], "ibo": ["biko", "nna", "nne", "chai", "ehen", "ngwa"],
+        "hau": ["to", "kai", "wallahi", "haba"]}
+
+_T = {}
+def tagger(lid="glotlid"):
+    if lid not in _T:
+        try:
+            _T[lid] = Tagger(lid if lid != "none" else None)
+        except Exception as e:  # no fasttext / GlotLID: rules only
+            print("tagger: GlotLID unavailable, rules only:", repr(e)[:120]); _T[lid] = Tagger(None)
+    return _T[lid]
+
+def _find(fs, phrases):
+    for p in sorted(phrases, key=len, reverse=True):
+        if re.search(r"(^|\s)" + re.escape(p) + r"(\s|$)", fs):
+            return p
+    return None
+
+PCM_MARK = {"dey", "don", "abeg", "wetin", "una", "gree", "wahala", "sabi", "comot", "chop", "oya", "abi", "shey", "wey", "dem", "wan",
+            "pikin", "oga", "sef", "jare", "naim", "dis", "dat", "wahala", "kpatakpata", "ehn", "sha", "nor", "abeg", "jor", "ehen", "na"}
+PCM_BI = {("no", "gree"), ("no", "be"), ("e", "no"), ("make", "we"), ("how", "far"), ("na", "im"), ("i", "go"), ("you", "go"), ("we", "go"), ("e", "go"),
+          ("no", "go"), ("i", "fit"), ("you", "fit"), ("e", "don"), ("i", "don"), ("my", "guy"), ("na", "so"), ("well", "well"), ("no", "dey")}
+
+def pidgin_relabel(words, labs, text):
+    """sentence-level Pidgin: a sentence of English-labelled words with Pidgin markers is Pidgin (the word tagger only flips the marker's clause)."""
+    if not labs: return labs
+    labs = list(labs); f = [fold(w).strip(".,!?;:'\"") for w in words]
+    ends = [i for i, w in enumerate(words) if re.search(r"[.!?]$", w)] + [len(words) - 1]
+    s0 = 0
+    for e in sorted(set(ends)):
+        idx = list(range(s0, e + 1)); s0 = e + 1
+        if not idx: continue
+        hits = sum(1 for i in idx if f[i] in PCM_MARK or f[i] == "omo") + sum(1 for a, b in zip(idx, idx[1:]) if (f[a], f[b]) in PCM_BI and not ((f[a], f[b]) == ("how", "far") and a != idx[0]))
+        hits += 1 if f[idx[-1]] == "o" and len(idx) > 2 else 0
+        lat = [i for i in idx if labs[i] in ("eng", "pcm")]
+        if hits == 0 and len(lat) > 4 and all(labs[i] == "pcm" for i in lat):
+            for i in lat: labs[i] = "eng"
+            continue
+        if lat and (hits >= 2 or (hits >= 1 and len(idx) <= 8)) and sum(labs[i] == "pcm" for i in lat) < len(lat):
+            for i in lat: labs[i] = "pcm"
+    return labs
+
+def window_relabel(words, labs, t, win=4, thr=0.5):
+    """ASR text has no punctuation/diacritics, so the clause-level tagger sees one long 'English' clause. Slide a GlotLID window
+    over the words and let confident window votes override the rule labels (rule label keeps a weight-1 vote)."""
+    from collections import Counter
+    n = len(words)
+    if t.lid is None or n < 3: return labs
+    votes = [Counter({labs[i]: 1.0}) for i in range(n)]
+    for s0 in range(0, max(1, n - win + 1)):
+        sc = t._lid_scores(" ".join(words[s0:s0 + win])); l = max(sc, key=sc.get)
+        if sc[l] >= thr:
+            for i in range(s0, min(n, s0 + win)): votes[i][l] += sc[l]
+    return [v.most_common(1)[0][0] for v in votes]
+
+def estimate(text, lid="glotlid", prior=None, prior_w=0.0, asr=False):
+    """prior: a previous mix dict (conversation-level style), blended with weight prior_w."""
+    t = tagger(lid)
+    toks, labs = t.word_langs(text) if text.strip() else ([], [])
+    words = [w for _, _, w in toks]
+    if asr: labs = window_relabel(words, list(labs), t)
+    labs = pidgin_relabel(words, labs, text)
+    n = max(1, len(labs))
+    mix = {l: labs.count(l) / n for l in LANGS if labs.count(l)}
+    if prior and prior_w > 0:
+        keys = set(mix) | set(prior)
+        mix = {k: (1 - prior_w) * mix.get(k, 0) + prior_w * prior.get(k, 0) for k in keys}
+        s = sum(mix.values()) or 1; mix = {k: v / s for k, v in mix.items() if v / s >= 0.03}
+    fs = " " + re.sub(r"[^\w\s']", " ", fold(text)) + " "
+    fs = re.sub(r"\s+", " ", fs)
+    pat = []
+    head = " ".join(fs.split()[:5]); tail = " ".join(fs.split()[-5:])
+    for l in ("ibo", "yor", "hau", "pcm"):
+        g = _find(head, GREET[l]); c = _find(tail, CLOSE[l])
+        if g: pat.append(f"greeting:{l}")
+        if c: pat.append(f"closing:{l}")
+        d = [p for p in DISC[l] if re.search(r"(^|\s)" + re.escape(p) + r"(\s|$)", fs)]
+        if d and l in mix: pat.append(f"discourse:{l}")
+    # ASR often spells a greeting/closing so that the word tagger calls it English: give the matched words to that language
+    toks_f = fs.split()
+    for p in pat:
+        k, _, l = p.partition(":")
+        if k in ("greeting", "closing") and mix.get(l, 0) < 0.1 and labs:
+            phr = _find(head if k == "greeting" else tail, (GREET if k == "greeting" else CLOSE)[l]) or ""
+            share = min(0.5, len(phr.split()) / max(1, len(labs)))
+            m2 = max(mix, key=mix.get)
+            mix[m2] = max(0.0, mix[m2] - share); mix[l] = mix.get(l, 0) + share
+            mix = {a: b for a, b in mix.items() if b > 0.01}
+    # switch style: count switches between labelled runs
+    sw = sum(1 for a, b in zip(labs, labs[1:]) if a != b)
+    if len(mix) > 1:
+        pat.append("intra-sentential" if sw >= 3 else "inter-sentential")
+    matrix = max(mix, key=mix.get) if mix else "eng"
+    return dict(mix={k: round(v, 2) for k, v in sorted(mix.items(), key=lambda x: -x[1])}, matrix=matrix, pattern=pat,
+                words=words, labels=labs, n_words=len(labs), switches=sw)
+
+def describe(est):
+    """natural-language mix instruction for the LLM."""
+    mix, pat = est["mix"], est["pattern"]
+    if not mix:
+        return "Reply in English."
+    parts = [f"{NAMES[l]} about {round(100 * p / 5) * 5}%" for l, p in mix.items()]
+    s = f"Match the user's language mix: {', '.join(parts)} of the words. The main (matrix) language is {NAMES[est['matrix']]}."
+    for p in pat:
+        k, _, l = p.partition(":")
+        if k == "greeting": s += f" The user greeted in {NAMES[l]}: open with a short {NAMES[l]} greeting."
+        if k == "closing": s += f" The user closed in {NAMES[l]}: you may close in {NAMES[l]}."
+        if k == "discourse": s += f" Sprinkle a {NAMES[l]} discourse word or two the way the user did."
+    if "intra-sentential" in pat: s += " The user switches inside sentences; do the same naturally."
+    if len(mix) == 1: s += f" Reply only in {NAMES[est['matrix']]}."
+    return s
+
+def tvd(a, b):
+    k = set(a) | set(b)
+    return 0.5 * sum(abs(a.get(x, 0) - b.get(x, 0)) for x in k)
+
+if __name__ == "__main__":
+    import sys, json
+    for s in sys.argv[1:] or ["Ndewo! How are you doing today? I wan ask you something.", "Ẹ káàárọ̀, ṣé dáadáa ni? I need help with my account."]:
+        e = estimate(s); print(json.dumps({k: e[k] for k in ("mix", "matrix", "pattern")}, ensure_ascii=False), "|", describe(e))

code/offline.py

@@ -0,0 +1,124 @@
+"""Offline duplex driver: runs scripted conversations (or a batch eval) through the SAME components as the live server,
+on a simulated clock. Mic audio is fed in 32 ms frames; compute stages are timed on the real GPU and placed on the timeline,
+so latency = (end-of-turn silence wait) + ASR + mix/LID + LLM/validator + first TTS chunk.
+  python offline.py convo scenarios.json OUTDIR
+  python offline.py eval  list.jsonl OUTDIR [N]"""
+import os, sys, json, time, re, numpy as np, soundfile as sf, librosa, httpx
+from turn import TurnDetector, FR, SR, FMS
+from respond import Responder
+from tts_engine import TTS
+
+ASR_URL = os.environ.get("ASR_URL", "http://127.0.0.1:8002/asr")
+_cli = httpx.Client(timeout=30)
+def asr(y):
+    t = time.perf_counter(); r = _cli.post(ASR_URL, content=np.ascontiguousarray(y, np.float32).tobytes()).json()
+    return r["text"], time.perf_counter() - t
+
+BC = {"eng": "<eng>Mm-hmm.</eng>", "pcm": "<pcm>Ehen.</pcm>", "ibo": "<ibo>Ehen.</ibo>", "yor": "<yor>Ehen.</yor>", "hau": "<hau>To.</hau>"}
+def trim(w, sr, db=35):
+    yt, idx = librosa.effects.trim(w, top_db=db); return w[max(0, idx[0] - int(0.03 * sr)): idx[1] + int(0.05 * sr)]
+
+class Sim:
+    def __init__(self, tts, resp, eot_ms=600):
+        self.tts, self.resp, self.eot_ms = tts, resp, eot_ms
+        self.osr = tts.sr
+        self.bc = {k: trim(tts.synth(v, seed=3), self.osr) for k, v in BC.items()}
+        import mix as M; M.estimate("warm up the GlotLID model", asr=True)
+        import logging; logging.getLogger("httpx").setLevel(logging.WARNING); logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+    def run(self, turns, label=""):
+        td = TurnDetector(eot_ms=self.eot_ms)
+        T = 600; mic = np.zeros(int(T * SR), np.float32); agt = np.zeros(int(T * self.osr), np.float32)
+        f = 0; busy_until = 0.0; reply_start = None; hist = []; log = []; t_next = 0.5; cur_chunks = []; style_lang = "eng"
+        for k, tn in enumerate(turns):
+            u = tn["audio"]
+            if tn.get("barge_at") is not None and reply_start is not None:
+                start = reply_start + tn["barge_at"]
+            else:
+                start = max(t_next, busy_until + tn.get("gap", 0.7))
+            s0 = int(start * SR); mic[s0:s0 + len(u)] += u
+            e = np.abs(u) > 0.02; onset = start + (np.argmax(e) / SR if e.any() else 0)
+            rec = dict(turn=k, user_label=tn.get("label", ""), user_ref=tn.get("text", ""), user_start=round(start, 3), user_onset=round(onset, 3), backchannels=[])
+            ev_eot = None
+            while ev_eot is None:
+                fr = mic[f * FR:(f + 1) * FR]; f += 1; tnow = f * FMS
+                busy = tnow < busy_until
+                for name, info in td.feed(fr, agent_busy=busy):
+                    if name == "barge" and busy:
+                        t0 = time.perf_counter(); stop_overhead = time.perf_counter() - t0 + 0.040   # + 40 ms output buffer flush
+                        t_stop = tnow + stop_overhead
+                        a0 = int(t_stop * self.osr); fade = int(0.02 * self.osr)
+                        agt[a0:a0 + fade] *= np.linspace(1, 0, fade)[: len(agt[a0:a0 + fade])]; agt[a0 + fade:] = 0
+                        played = [c for c in cur_chunks if c[1] < t_stop]
+                        if hist:
+                            hist[-1] = (hist[-1][0], " ".join(c[0] for c in played) + " [interrupted]")
+                        rec["barge"] = dict(user_onset=round(onset, 3), detected=round(tnow, 3), agent_stopped=round(t_stop, 3), reaction_s=round(t_stop - onset, 3),
+                                            agent_chunks_played=len(played), agent_chunks_total=len(cur_chunks))
+                        busy_until = t_stop
+                    elif name == "backchannel" and not busy:
+                        w = self.bc.get(style_lang, self.bc["eng"]); a0 = int((tnow + 0.05) * self.osr); agt[a0:a0 + len(w)] += 0.8 * w
+                        rec["backchannel_at"] = round(tnow, 3); rec["backchannels"].append(dict(t=round(tnow, 3), turn_s=round(info["turn_s"], 2)))
+                    elif name == "eot":
+                        ev_eot = (tnow, info)
+            T_eot, info = ev_eot
+            text, t_asr = asr(info["audio"])
+            t_start = info["speech_end"] + (td.spec_f * FMS if td.spec_f else (T_eot - info["speech_end"]))   # speculative start at spec_ms of silence
+            r = self.resp.reply(text, hist)
+            style_lang = r["target"]["matrix"]
+            t_cmp = t_asr + r["t_total"]
+            prev_end = 0; cum = 0; first = None; cur_chunks = []; tts_times = []
+            for c, w, ds in self.tts.stream(r["reply"], seed=k):
+                cum += ds; tts_times.append(round(ds, 3))
+                avail = max(T_eot, t_start + t_cmp + cum); ps = max(avail, prev_end)
+                a0 = int(ps * self.osr); agt[a0:a0 + len(w)] += w
+                prev_end = ps + len(w) / self.osr; cur_chunks.append((c, ps, prev_end))
+                if first is None: first = ps
+            reply_start = first; busy_until = prev_end; t_next = prev_end
+            hist.append((text, r["reply"]))
+            rec.update(asr=text, asr_s=round(t_asr, 3), eot_wait_s=round(T_eot - info["speech_end"], 3), speech_end=round(info["speech_end"], 3),
+                       target=r["target"], target_desc=r["target_desc"], reply=r["reply"], reply_mix=r["reply_mix"], tvd=r["tvd"], how=r["how"],
+                       first_cand_ok=r["first_cand_ok"], rounds=r["rounds"], mix_s=r["t_mix"], llm_s=r["t_llm"], tts_first_s=tts_times[0] if tts_times else None, spec_start_s=round(t_start - info["speech_end"], 3), compute_to_first_audio_s=round(t_cmp + (tts_times[0] if tts_times else 0), 3),
+                       tts_breakdown=getattr(self.tts, "last", None),
+                       tts_chunks_s=tts_times, first_audio=round(first, 3) if first else None, reply_end=round(prev_end, 3),
+                       latency_s=round(first - info["speech_end"], 3) if first else None)
+            log.append(rec); print(json.dumps({k2: rec[k2] for k2 in ("turn", "asr", "reply", "latency_s", "how")}, ensure_ascii=False), flush=True)
+        end = max(busy_until, f * FMS) + 0.6
+        mic24 = librosa.resample(mic[: int(end * SR)], orig_sr=SR, target_sr=self.osr)
+        n = min(len(mic24), int(end * self.osr)); a = agt[:n]; m = mic24[:n]
+        return dict(log=log, mono=np.clip(0.9 * (m + a), -1, 1), stereo=np.stack([m, a], 1), sr=self.osr)
+
+def user_audio(tts, tn):
+    if tn.get("wav"):
+        y, sr = sf.read(tn["wav"], dtype="float32"); y = y.mean(1) if y.ndim > 1 else y
+        return librosa.resample(y, orig_sr=sr, target_sr=SR) if sr != SR else y
+    parts = tn["text_parts"] if "text_parts" in tn else [tn["text"]]
+    ws = []
+    for i, p in enumerate(parts):
+        w = trim(tts.synth(p, seed=11 + i, voice=tn.get("voice", "waxal_ibo_5_female")), tts.sr)
+        ws += [w, np.zeros(int(tn.get("inner_pause", 0.35) * tts.sr), np.float32)]
+    y = np.concatenate(ws[:-1]); return librosa.resample(y, orig_sr=tts.sr, target_sr=SR).astype(np.float32) * 0.9
+
+def save(out, name, res, meta):
+    os.makedirs(out, exist_ok=True)
+    sf.write(f"{out}/{name}.wav", res["mono"], res["sr"]); sf.write(f"{out}/{name}_stereo_userL_agentR.wav", res["stereo"], res["sr"])
+    json.dump(dict(meta, turns=res["log"]), open(f"{out}/{name}.json", "w"), ensure_ascii=False, indent=1, default=str)
+
+if __name__ == "__main__":
+    mode, src, out = sys.argv[1], sys.argv[2], sys.argv[3]
+    tts = TTS(os.environ.get("AGENT_VOICE", "anchor_male")); resp = Responder(); sim = Sim(tts, resp)
+    if mode == "convo":
+        for sc in json.load(open(src)):
+            resp.style = None
+            turns = [dict(t, audio=user_audio(tts, t)) for t in sc["turns"]]
+            res = sim.run(turns, sc["name"]); save(out, sc["name"], res, {k: v for k, v in sc.items() if k != "turns"} | dict(user_turns=[{k: v for k, v in t.items() if k != "audio"} for t in turns]))
+    else:   # eval: single-turn items from the ASR eval list (real held-out audio), each followed by a barge-in at 1.0 s
+        R = [json.loads(l) for l in open(src)]; N = int(sys.argv[4]) if len(sys.argv) > 4 else 40
+        import random; random.seed(1); R = random.sample(R, min(N, len(R))); allr = []
+        for i, r in enumerate(R):
+            resp.style = None
+            barger = R[(i + 1) % len(R)]
+            turns = [dict(wav=r["wav"], label=r["id"], text=r["text"]), dict(wav=barger["wav"], label=barger["id"] + " (barge-in)", text=barger["text"], barge_at=1.0)]
+            turns = [dict(t, audio=user_audio(tts, t)) for t in turns]
+            res = sim.run(turns, r["id"]); allr.append(dict(id=r["id"], lang=r["lang"], ref=r["text"], turns=res["log"]))
+            json.dump(allr, open(f"{out}/eval_turns.json", "w"), ensure_ascii=False, indent=1, default=str)
+    print("OFFLINE_DONE")

code/respond.py

@@ -0,0 +1,35 @@
+"""Responder: ASR text -> mix estimate -> LLM candidates -> validator (regenerate up to N rounds) -> tagged reply."""
+import time, re
+import mix as M, validator as V
+from llm import LLM, build_messages
+
+class Responder:
+    def __init__(self, llm=None, n=3, rounds=2, lid="glotlid", prior_w=0.3, asr=True):
+        self.llm = llm or LLM(); self.n, self.rounds, self.lid, self.prior_w = n, rounds, lid, prior_w
+        self.asr = asr; self.style = None          # running conversation-level mix (user's style)
+
+    def reply(self, user_text, history=()):
+        t0 = time.perf_counter()
+        est = M.estimate(user_text, lid=self.lid, prior=self.style, prior_w=self.prior_w if self.style else 0.0, asr=self.asr)
+        cur = M.estimate(user_text, lid=self.lid, asr=self.asr)
+        self.style = cur["mix"] if self.style is None else {k: 0.5 * self.style.get(k, 0) + 0.5 * cur["mix"].get(k, 0) for k in set(self.style) | set(cur["mix"])}
+        target = dict(mix=est["mix"], matrix=est["matrix"], pattern=cur["pattern"])
+        desc = M.describe(dict(target, mix=est["mix"]))
+        t_mix = time.perf_counter() - t0
+        cands, log, t_llm = [], [], 0.0
+        chosen, how = None, None
+        for rd in range(self.rounds):
+            msgs = build_messages(user_text, desc + ("" if rd == 0 else " IMPORTANT: your previous attempt did not follow the language target or the tag format. Follow it exactly."), history)
+            outs, dt = self.llm.generate(msgs, n=self.n, temperature=0.7 if rd == 0 else 0.9)
+            t_llm += dt
+            rs = [V.check(o, target, lid=self.lid, user=user_text) for o in outs]
+            cands += rs; log.append(dict(round=rd, llm_s=round(dt, 3), cands=[dict(text=r["text"], ok=r["ok"], why=r["why"], tvd=r["tvd"]) for r in rs]))
+            ok = [r for r in rs if r["ok"]]
+            if ok:
+                chosen = min(ok, key=lambda r: r["score"]); how = "first_round" if rd == 0 else f"regen_round{rd}"
+                break
+        if chosen is None:
+            text, how = V.fallback(cands, target); chosen = V.check(text, target, lid=self.lid); chosen["text"] = text
+        return dict(user=user_text, target=target, target_desc=desc, reply=chosen["text"], reply_mix=chosen.get("mix", {}),
+                    tvd=chosen.get("tvd"), how=how, first_cand_ok=bool(cands and cands[0]["ok"]), any_first_round_ok=any(r["ok"] for r in cands[:self.n]),
+                    rounds=log, t_mix=round(t_mix, 4), t_llm=round(t_llm, 3), t_total=round(time.perf_counter() - t0, 3))

code/scenarios.json

@@ -0,0 +1,33 @@
+[
+ {"name": "01_igbo_greetings_english", "desc": "Mostly English with Igbo greetings/closing (ACE's target case). Includes a long turn (backchannel) and a barge-in. User turns are TTS-synthesized (Flash, female Igbo voice).",
+  "turns": [
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<ibo>Ndewo!</ibo> <eng>Good morning. Please, can you help me plan my day? I have a lot to do.</eng>"},
+   {"label": "TTS-synthesized user turn (long, mid-turn pause)", "voice": "waxal_ibo_5_female", "text_parts": ["<eng>I need to go to the bank, and then the market,</eng>", "<eng>and in the evening I also want to call my mother.</eng>"], "inner_pause": 0.4},
+   {"label": "TTS-synthesized user turn (barge-in 1.5 s into the reply)", "voice": "waxal_ibo_5_female", "text": "<eng>Wait, sorry, the bank closes early today.</eng>", "barge_at": 1.5},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<eng>Okay, thank you so much.</eng> <ibo>Daalụ!</ibo>"}]},
+ {"name": "02_yoruba_greetings_english", "desc": "Yoruba greetings + English. User turns TTS-synthesized (Flash, female Yoruba voice).",
+  "turns": [
+   {"label": "TTS-synthesized user turn", "voice": "waxal_yor_5_female", "text": "<yor>Ẹ káàárọ̀!</yor> <eng>I want to buy a new phone for my sister.</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_yor_5_female", "text": "<eng>Her budget is about one hundred thousand naira.</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_yor_5_female", "text": "<eng>Okay, that sounds good.</eng> <yor>Ẹ ṣé!</yor>"}]},
+ {"name": "03_pidgin_bargein", "desc": "Nigerian Pidgin throughout, with a barge-in. User turns TTS-synthesized (Flash, female voice).",
+  "turns": [
+   {"label": "TTS-synthesized user turn", "voice": "waxal_yor_5_female", "text": "<pcm>How far? Abeg, wetin I fit cook tonight wey no go take long?</pcm>"},
+   {"label": "TTS-synthesized user turn (barge-in 1.2 s into the reply)", "voice": "waxal_yor_5_female", "text": "<pcm>No be rice o, I no get rice for house.</pcm>", "barge_at": 1.2},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_yor_5_female", "text": "<pcm>Okay, na so. Thank you well well.</pcm>"}]},
+ {"name": "04_hausa_greetings_english", "desc": "Hausa greeting + English, Hausa closing. User turns TTS-synthesized (Flash, female voice).",
+  "turns": [
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<hau>Sannu, barka da yamma.</hau> <eng>I am travelling to Kano tomorrow.</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<eng>What should I pack for the weather there?</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<hau>To, na gode.</hau>"}]},
+ {"name": "05_igbo_heavy", "desc": "Igbo-heavier user (shows the weak spot: LLM Igbo beyond phrases). User turns TTS-synthesized (Flash, female Igbo voice).",
+  "turns": [
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<ibo>Kedu? Biko, nyere m aka.</ibo> <eng>I need a taxi to the airport.</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<ibo>Ee, ọ dị mma.</ibo> <eng>I will leave at six.</eng>"},
+   {"label": "TTS-synthesized user turn", "voice": "waxal_ibo_5_female", "text": "<ibo>Daalụ, ka ọ dị.</ibo>"}]},
+ {"name": "06_real_waxal_clips", "desc": "User turns are REAL held-out WAXAL test recordings (CC-BY-SA-4.0; read sentences, so the content is not conversational): Pidgin, an English+Igbo concatenation, Hausa.",
+  "turns": [
+   {"label": "real WAXAL test clip pcm_121", "wav": "/scratch/duplex/asr_eval/wav/pcm_121.wav"},
+   {"label": "real WAXAL/FLEURS clips concatenated (eng + ibo), cs_000", "wav": "/scratch/duplex/asr_eval/wav/cs_000.wav"},
+   {"label": "real WAXAL test clip hau_081", "wav": "/scratch/duplex/asr_eval/wav/hau_081.wav"}]}
+]

code/server.py

@@ -0,0 +1,132 @@
+"""Live full-duplex demo: browser mic (16 kHz PCM over a WebSocket) -> VAD/turn-taking -> streaming ASR (partials) ->
+mix-matched LLM reply + validator -> Flash-DPO TTS streamed back sentence by sentence; barge-in stops playback at once.
+  ASR_URL=http://127.0.0.1:8002/asr LLM_BASE=http://127.0.0.1:8001/v1 python server.py  (port 7860)"""
+import asyncio, json, os, time, threading, re
+import numpy as np, httpx, uvicorn
+from fastapi import FastAPI, WebSocket, WebSocketDisconnect
+from fastapi.responses import HTMLResponse
+from turn import TurnDetector, FR, SR, FMS
+from respond import Responder
+from tts_engine import TTS
+from offline import BC, trim
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ASR_URL = os.environ.get("ASR_URL", "http://127.0.0.1:8002/asr")
+app = FastAPI()
+TTS_ = TTS(os.environ.get("AGENT_VOICE", "anchor_male"))
+LOCK = threading.Lock()
+BCW = {k: trim(TTS_.synth(v, seed=3), TTS_.sr) for k, v in BC.items()}
+acli = httpx.AsyncClient(timeout=30)
+import mix as _M, logging; _M.estimate('warm up', asr=True); logging.getLogger('httpx').setLevel(logging.WARNING)
+
+def pcm16(w):
+    return (np.clip(w, -1, 1) * 32767).astype(np.int16).tobytes()
+
+async def asr(y):
+    r = await acli.post(ASR_URL, content=np.ascontiguousarray(y, np.float32).tobytes())
+    return r.json()
+
+def synth_locked(c, seed):
+    with LOCK:
+        t = time.perf_counter(); w = TTS_.synth(c, seed=seed); return w, time.perf_counter() - t
+
+@app.get("/")
+def index():
+    return HTMLResponse(open(f"{HERE}/static/index.html", encoding="utf-8").read())
+
+@app.websocket("/ws")
+async def ws(w: WebSocket):
+    await w.accept()
+    loop = asyncio.get_running_loop()
+    td = TurnDetector(eot_ms=int(os.environ.get("EOT_MS", 600))); resp = Responder(); hist = []
+    st = dict(play_until=0.0, task=None, gen=0, last_partial=0.0, partial_busy=False, lang="eng")
+    pending = bytearray()
+    from tts_engine import chunk_tagged
+
+    async def partial(y):
+        st["partial_busy"] = True
+        try:
+            r = await asr(y); await w.send_json({"type": "partial", "text": r["text"], "asr_ms": r["ms"]})
+        except Exception: pass
+        st["partial_busy"] = False
+
+    gate = {}   # gen -> asyncio.Event set at end-of-turn (speculative responses start at spec_ms of silence)
+
+    async def speak(audio, speech_end_wall, gen):
+        """ASR -> mix/LLM/validator -> first TTS chunk are computed speculatively (from spec_ms of silence); nothing is sent
+        until end-of-turn opens the gate, and the whole response is dropped if the user resumes (gen changes)."""
+        r = await asr(audio); text = r["text"].strip()
+        if len(text.split()) == 0 or st["gen"] != gen:
+            return
+        rep = await loop.run_in_executor(None, resp.reply, text, list(hist))
+        chunks = chunk_tagged(rep["reply"])
+        if st["gen"] != gen or not chunks:
+            return
+        wav0, ds0 = await loop.run_in_executor(None, synth_locked, chunks[0], 0)
+        if gen in gate:
+            await gate[gen].wait()
+        if st["gen"] != gen:
+            return
+        st["committed"] = gen; st["lang"] = rep["target"]["matrix"]
+        await w.send_json({"type": "final", "text": text, "asr_ms": r["ms"]})
+        await w.send_json({"type": "reply", "text": rep["reply"], "target": rep["target"], "reply_mix": rep["reply_mix"], "how": rep["how"],
+                           "llm_s": rep["t_llm"]})
+        played = []
+        for i, c in enumerate(chunks):
+            if i == 0:
+                wav, ds = wav0, ds0
+            else:
+                wav, ds = await loop.run_in_executor(None, synth_locked, c, i)
+            if st["gen"] != gen: break
+            await w.send_bytes(pcm16(wav))
+            now = time.time(); st["play_until"] = max(st["play_until"], now) + len(wav) / TTS_.sr; played.append(c)
+            if i == 0:
+                await w.send_json({"type": "metrics", "latency_s": round(now - speech_end_wall, 3), "asr_ms": r["ms"], "llm_s": rep["t_llm"],
+                                   "tts_first_s": round(ds, 3), "eot_ms": round(td.eot_f * FMS * 1000)})
+        hist.append((text, " ".join(played) + ("" if len(played) == len(chunks) else " [interrupted]")))
+        if st.get("committed") == gen:
+            st["committed"] = None
+
+    try:
+        while True:
+            msg = await w.receive()
+            if msg.get("type") == "websocket.disconnect": break
+            if msg.get("text"):
+                d = json.loads(msg["text"])
+                if d.get("type") == "reset": hist.clear(); resp.style = None; await w.send_json({"type": "info", "text": "conversation reset"})
+                continue
+            if not msg.get("bytes"): continue
+            pending += msg["bytes"]
+            while len(pending) >= FR * 2:
+                fr = np.frombuffer(bytes(pending[:FR * 2]), np.int16).astype(np.float32) / 32768; del pending[:FR * 2]
+                busy = time.time() < st["play_until"] or st.get("committed") is not None
+                for name, info in td.feed(fr, agent_busy=busy):
+                    if name == "start":
+                        await w.send_json({"type": "vad", "speech": True})
+                    elif name == "barge":
+                        st["gen"] += 1; st["play_until"] = 0.0; st["committed"] = None
+                        await w.send_json({"type": "stop", "reason": "barge-in"})
+                    elif name == "backchannel":
+                        await w.send_json({"type": "backchannel"}); await w.send_bytes(pcm16(0.8 * BCW.get(st["lang"], BCW["eng"])))
+                    elif name == "spec":
+                        st["gen"] += 1; g = st["gen"]; gate[g] = asyncio.Event()
+                        speech_end_wall = time.time() - (td.t - info["speech_end"])
+                        st["task"] = asyncio.create_task(speak(info["audio"], speech_end_wall, g))
+                    elif name == "resume":
+                        if st["gen"] in gate: gate[st["gen"]].set()     # release the waiting speculative task; it sees the new gen and exits
+                        st["gen"] += 1          # user kept talking: drop the speculative response
+                    elif name == "eot":
+                        await w.send_json({"type": "vad", "speech": False, "speech_s": round(info["speech_s"], 2)})
+                        if st["gen"] in gate:
+                            gate[st["gen"]].set()
+                        else:
+                            st["gen"] += 1
+                            speech_end_wall = time.time() - (td.t - info["speech_end"])
+                            st["task"] = asyncio.create_task(speak(info["audio"], speech_end_wall, st["gen"]))
+                if td.in_speech and td.t - st["last_partial"] >= 0.64 and not st["partial_busy"]:
+                    st["last_partial"] = td.t; asyncio.create_task(partial(td.partial_audio()))
+    except WebSocketDisconnect:
+        pass
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)), log_level="warning")

code/static/index.html

@@ -0,0 +1,46 @@
+<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Dynamic Accent Duplex (experimental)</title>
+<style>body{font-family:system-ui,sans-serif;max-width:760px;margin:24px auto;padding:0 14px;color:#222}
+button{font-size:16px;padding:10px 18px;margin-right:8px;border-radius:8px;border:1px solid #888;background:#f4f4f4;cursor:pointer}
+#log{border:1px solid #ddd;border-radius:8px;padding:10px;height:420px;overflow:auto;font-size:14px;background:#fafafa}
+.u{color:#0a4}.a{color:#036}.p{color:#999;font-style:italic}.m{color:#a60;font-size:12px}.s{color:#c00}
+#dot{display:inline-block;width:12px;height:12px;border-radius:6px;background:#ccc;vertical-align:middle;margin-left:6px}
+small{color:#666}</style></head><body>
+<h2>Dynamic Accent Duplex <small>experimental cascaded voice agent</small></h2>
+<p>Speak English, Nigerian Pidgin, Igbo, Yoruba or Hausa, mixed however you like. Ada answers in the same mix. You can interrupt her.
+<b>Use headphones</b> (otherwise her voice can trigger barge-in).</p>
+<button id="go">Start</button><button id="rs">Reset conversation</button> mic<span id="dot"></span>
+<div id="log"></div>
+<p><small>Pipeline: Silero VAD &rarr; Omnilingual ASR (CTC) &rarr; per-word language ID &rarr; Qwen3 LLM with mix validator &rarr; Dynamic Accent Flash TTS. Research demo; replies can be wrong, and Igbo/Yoruba/Hausa beyond simple phrases are weak.</small></p>
+<script>
+const log=(c,t)=>{const d=document.getElementById('log');const e=document.createElement('div');e.className=c;e.textContent=t;d.appendChild(e);d.scrollTop=1e9;return e};
+const strip=t=>t.replace(/<\/?(eng|ibo|yor|hau|pcm)>/g,m=>m.startsWith('</')?'':'['+m.slice(1,-1)+'] ');
+let ws,ctx,next=0,srcs=[],partialEl=null;
+function play(buf){const i16=new Int16Array(buf);const f=new Float32Array(i16.length);for(let i=0;i<i16.length;i++)f[i]=i16[i]/32768;
+ const b=ctx.createBuffer(1,f.length,24000);b.copyToChannel(f,0);const s=ctx.createBufferSource();s.buffer=b;s.connect(ctx.destination);
+ const t=Math.max(ctx.currentTime+0.02,next);s.start(t);next=t+b.duration;srcs.push(s);s.onended=()=>{srcs=srcs.filter(x=>x!==s)}}
+function stopAll(){srcs.forEach(s=>{try{s.stop()}catch(e){}});srcs=[];next=0}
+document.getElementById('rs').onclick=()=>{ws&&ws.send(JSON.stringify({type:'reset'}));document.getElementById('log').innerHTML=''};
+document.getElementById('go').onclick=async()=>{
+ ctx=new AudioContext({sampleRate:16000});
+ const st=await navigator.mediaDevices.getUserMedia({audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true,channelCount:1}});
+ const code=`class P extends AudioWorkletProcessor{constructor(){super();this.b=new Float32Array(512);this.n=0}
+ process(i){const x=i[0][0];if(!x)return true;for(let k=0;k<x.length;k++){this.b[this.n++]=x[k];if(this.n==512){const o=new Int16Array(512);
+ for(let j=0;j<512;j++){o[j]=Math.max(-32768,Math.min(32767,this.b[j]*32767))}this.port.postMessage(o.buffer,[o.buffer]);this.n=0}}return true}}
+ registerProcessor('p',P)`;
+ await ctx.audioWorklet.addModule(URL.createObjectURL(new Blob([code],{type:'application/javascript'})));
+ const node=new AudioWorkletNode(ctx,'p');ctx.createMediaStreamSource(st).connect(node);
+ ws=new WebSocket((location.protocol==='https:'?'wss':'ws')+'://'+location.host+location.pathname.replace(/\/$/,'')+'/ws');ws.binaryType='arraybuffer';
+ node.port.onmessage=e=>{if(ws.readyState===1)ws.send(e.data)};
+ ws.onopen=()=>log('m','connected - start talking');
+ ws.onmessage=e=>{if(typeof e.data!=='string'){play(e.data);return}const d=JSON.parse(e.data);
+  if(d.type==='vad'){document.getElementById('dot').style.background=d.speech?'#0a4':'#ccc'}
+  else if(d.type==='partial'){if(!partialEl)partialEl=log('p','');partialEl.textContent='… '+d.text}
+  else if(d.type==='final'){if(partialEl){partialEl.remove();partialEl=null}log('u','You: '+d.text)}
+  else if(d.type==='reply'){log('a','Ada: '+strip(d.text));log('m','target mix '+JSON.stringify(d.target.mix)+' '+(d.target.pattern||[]).join(',')+' | reply mix '+JSON.stringify(d.reply_mix)+' | '+d.how)}
+  else if(d.type==='metrics'){log('m','latency end-of-speech→first audio '+d.latency_s+' s (EOT wait '+d.eot_ms+' ms, ASR '+d.asr_ms+' ms, LLM '+d.llm_s+' s, TTS first chunk '+d.tts_first_s+' s)')}
+  else if(d.type==='stop'){stopAll();log('s','[barge-in: Ada stopped]')}
+  else if(d.type==='backchannel'){log('m','(backchannel)')}
+  else if(d.type==='info'){log('m',d.text)}};
+ ws.onclose=()=>log('s','disconnected');document.getElementById('go').disabled=true};
+</script></body></html>

code/tagger.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+"""Dynamic Accent auto-tagger: word/span-level language ID for untagged mixed Nigerian text.
+
+Labels: eng (English), pcm (Nigerian Pidgin), ibo (Igbo), yor (Yoruba), hau (Hausa).
+Output is the tagged text infer.py expects, e.g.
+    "omo, I'm very excited"  ->  "<pcm>omo,</pcm> <eng>I'm very excited</eng>"
+
+How it decides (rule-based, no model needed):
+  1. letters: Yoruba-only (ẹ ṣ, tone-marked n/m), Igbo-only (ị ụ ṅ), Hausa-only (ɓ ɗ ƙ ƴ), shared ọ;
+  2. lexicons of frequent function words / greetings per language (diacritics folded for lookup);
+  3. Pidgin markers ("dey", "abi", "omo", "wetin", "abeg", "una", "no go", "e don", "know say", pronoun + "go" + verb ...):
+     a clause of plain-ASCII words that contains a Pidgin marker becomes Pidgin as a whole;
+  4. clause context: ambiguous words ("na", "o", "ka", "ya", "ma", "ni" ...) and unknown words take the language of the
+     clause they sit in, then of the neighbouring clause; plain ASCII with no evidence falls back to English.
+  5. optional: --lid glotlid adds clause-level scores from GlotLID v3 (cis-lmu/glotlid, fastText, Apache-2.0 + notices,
+     1.7 GB download, needs `pip install fasttext`), mainly to separate untagged Igbo/Yoruba/Hausa written without
+     diacritics from English.
+
+Overrides: lang_map {"omo": "pcm"} forces a word; force_lang renders everything in one language.
+Always print the tagged text and let the user fix it: the tagger is a convenience, not ground truth.
+
+    python tagger.py "Omo, this traffic no be here o. I go call you later."
+    python tagger.py --lang_map "omo=pcm,sha=pcm" "..."
+"""
+import argparse, json, re, sys, unicodedata
+
+LANGS = ["eng", "pcm", "ibo", "yor", "hau"]
+NG = ("ibo", "yor", "hau")
+TAGGED = re.compile(r"<(eng|ibo|hau|yor|pcm)>")
+
+
+def fold(w):
+    """lowercase, NFD, drop every combining mark (tones, dot below, dot above), map hooked letters."""
+    w = unicodedata.normalize("NFD", w.lower())
+    w = "".join(c for c in w if not unicodedata.combining(c))
+    return w.translate(str.maketrans({"ɓ": "b", "ɗ": "d", "ƙ": "k", "ƴ": "y", "’": "'", "ʼ": "'", "‘": "'"}))
+
+
+def _set(s):
+    return {fold(x) for x in s.split()}
+
+
+def _exact(s):
+    return {unicodedata.normalize("NFC", x.lower()) for x in s.split()}
+
+
+# --- lexicons (frequent, high-precision words; folded). Words shared across languages are listed in each.
+_RAW = {
+    "ibo": """
+        bụ nke ndị ihe ụlọ mmiri kedu ndewo nụ ụmụ ụmụnna nna nne anyị unu gị ha ọ ya ka ma
+        nwere ga ji dị ọma biko daalụ dalu ụtọ obi ike nri ego akwụkwọ ọrụ maka ebe otu ọzọ ugbu
+        a m n' na mere meela kwa kwesịrị ịga ịbịa bịa gaa nyere aka mmadụ obodo nwanne nwa nwoke nwanyị
+        ụbọchị echi taa ụnyaahụ abalị ututu ehihie mgbede oge afọ ọnwa izu ukwuu nnukwu ntakịrị
+        mana ọbụna ọbụla ruo site gbasara ebee onye gịnị olee ole kpọmkwem eziokwu nso anọ abụọ atọ
+        ise isii asaa asatọ itoolu iri mbụ zuru ezu ahụ ahụla ịhụ hụ chere chọrọ chọ mara amaghị
+        nọ nọrọ bịara gara kwuru kwuo sị siri esi na-esi na-eme eme mee eji ogologo ọkụ ụzọ ahịa ụka
+        ọnụ ụtụtụ ka ọ dị mma mma nnọọ kpọọ zụta re ree rie riri ụmụaka ezinụlọ chi chukwu ndụ
+        """,
+    "yor": """
+        ni ti si sí àti àwọn mo o wa wọn kò ṣé jẹ́ ọjọ́ ẹ káàbọ̀ ilé ìlú ọmọ púpọ̀ dáadáa bẹ́ẹ̀ ni
+        náà yìí kan mi wa yín rẹ̀ rẹ tirẹ̀ fún láti pé ń máa yóò ó a jẹ ṣe lọ wá rí fẹ́ mọ̀ sọ gbọ́
+        ẹ̀kọ́ iṣẹ́ owó ọjà òní ọ̀la àná ìpàdé padà ilé-ìwé ọjọ́ ajé ìsinmi alẹ́ àárọ̀ káàárọ̀ káàsán
+        káalẹ́ ẹ kú ṣé gan an pàdé ìyá bàbá ẹ̀gbọ́n àbúrò ọ̀rẹ́ ènìyàn èèyàn àgbà ọba ìjọba
+        nítorí ṣùgbọ́n tàbí bí kí kì ìgbà ibi ibo kílódé báwo kíni ta tani mélòó ọ̀kan méjì mẹ́ta
+        mẹ́rin márùn-ún ẹ̀ẹ̀kan dúpẹ́ ẹ ṣeun ẹ̀ jọ̀ọ́ jọ̀ọ́ oúnjẹ omi ilẹ̀ ojú ọwọ́ ẹsẹ̀
+        """,
+    "hau": """
+        da ba ya ta ka ki su mu na ne ce ina yaya barka kuke don haka za idan mai wannan wancan yana
+        tana suna muna kuna sannu lafiya kwana aiki gobe yau jiya ruwa sai amma kuma shi ita nan
+        wani wata abin zan zai zata zamu mutane gida kudi gode sosai tafi fita zuwa laima yamma safe
+        rana dare mako shekara wata yanzu kai ke ku mun sun ban bai bata sani so iya yi yin
+        farashin tumatur tashi kasuwa abinci ruwan sha babu akwai kamar saboda domin tare har tun
+        daga cikin waje gaba baya sama kasa me wane wace ina yaushe nawa daya biyu uku hudu biyar
+        """,
+    "pcm": """
+        dey abi omo wetin abeg una wahala sabi oga sef sha oya jare ehn ehen shey kpele pikin
+        chop comot waka belle palava wan wey dem don am siddon gist gbam yawa kampe sharp sharp
+        nawa wahala kolo japa ginger vex dash ogbonge padi paddy shine eye
+        """,
+    "eng": """
+        the a an and or but if then so because of to in on at for with from by about as into over
+        after before between through during without under again further here there when where why how
+        all any both each few more most other some such no nor not only own same than too very can
+        will just should now i me my we our you your he him his she her it its they them their what
+        which who whom this that these those am is are was were be been being have has had having do
+        does did doing would could might must shall may please thank thanks yes hello hi good morning
+        evening afternoon night today tomorrow yesterday week year time day people man woman child
+        children team meeting report school market road weather rain safe everyone everybody
+        new old big small great right left back home work job money food water house car family
+        friend friends know think say said tell told make made take took get got give gave go went
+        come came see saw look want need like love help call send sent finish start stop keep let put
+        read write talk speak discuss three two one four five six seven eight nine ten first second
+        last next soon late early ready sure also still really very much many lot thing things way
+        out up down off well done going coming doing nothing something everything anything model
+        """,
+}
+LEX = {l: _set(v) for l, v in _RAW.items()}          # folded (diacritics dropped) for lookup
+LEX_EXACT = {l: _exact(v) for l, v in _RAW.items()}  # as written
+_N_LEX = {}
+for _l, _v in LEX.items():
+    for _w in _v:
+        _N_LEX[_w] = _N_LEX.get(_w, 0) + 1          # >1 = word shared by several languages (na, o, ka, ya ...)
+# words that are Pidgin markers on their own (strong evidence)
+PCM_STRONG = _set("dey abi omo wetin abeg una wahala sabi sef oya jare shey pikin chop comot waka palava wan wey dem siddon vex japa ehn ehen")
+PCM_BIGRAMS = {("no", "go"), ("no", "be"), ("no", "fit"), ("no", "dey"), ("e", "don"), ("e", "dey"), ("e", "no"),
+               ("know", "say"), ("tell", "am"), ("well", "well"), ("how", "far"), ("how", "body"), ("na", "so"),
+               ("make", "we"), ("make", "i"), ("make", "una"), ("make", "dem"), ("i", "no"), ("we", "no"),
+               ("you", "no"), ("dem", "no"), ("no", "easy"), ("no", "forget"), ("no", "vex"), ("fit", "heavy"),
+               ("don", "come"), ("don", "go"), ("for", "inside"), ("una", "well"), ("e", "go"), ("na", "you"),
+               ("na", "me"), ("na", "im"), ("na", "dem"), ("na", "una"), ("give", "you"), ("lie", "give")}
+PRON = _set("i we you dem una e im him she he they")
+NOT_VERB_AFTER_GO = _set("to home there back out away on in up down off ahead")
+PCM_AMBIG_END = _set("o")          # clause-final particle "o" in an ASCII clause -> Pidgin evidence
+
+YOR_CH = set("ẹṣ")                   # after NFC lowercase; also e/s + U+0323
+IBO_CH = set("ịụṅ")
+HAU_CH = set("ɓɗƙƴ")
+WORD = re.compile(r"[^\W\d_](?:[^\W\d_]|['’ʼ\-\u0300-\u036f])*", re.U)
+CLAUSE_END = re.compile(r"[,;:.?!]")
+
+
+def char_scores(w):
+    s = dict.fromkeys(LANGS, 0.0)
+    nfd = unicodedata.normalize("NFD", w.lower())
+    base = None
+    for c in nfd:
+        if unicodedata.combining(c):
+            if c == "\u0323":                    # dot below
+                if base in "es":
+                    s["yor"] += 3
+                elif base in "iu":
+                    s["ibo"] += 3
+                elif base == "o":
+                    s["yor"] += 1; s["ibo"] += 1
+            elif c == "\u0307" and base == "n":
+                s["ibo"] += 3
+            elif c in "\u0300\u0301\u0304":       # tones
+                if base in "nm":
+                    s["yor"] += 2
+                else:
+                    s["yor"] += 0.8; s["ibo"] += 0.3
+            continue
+        base = c
+        if c in HAU_CH:
+            s["hau"] += 4
+    if not any(s.values()) and re.fullmatch(r"[a-z'\-]+", fold(w)):
+        s["ascii"] = 1
+    return s
+
+
+def word_scores(w):
+    s = char_scores(w)
+    ascii_ = s.pop("ascii", 0)
+    f = fold(w)
+    parts = [f] + (f.split("-") if "-" in f else [])
+    if f.startswith("n'") or re.match(r"^na-[a-z]", f):
+        s["ibo"] += 3
+    ex = unicodedata.normalize("NFC", w.lower())
+    for p in parts:
+        amb = 0.5 if _N_LEX.get(p, 0) > 1 else 1.0          # shared words: let the clause decide
+        for l in LANGS:
+            if p in LEX[l]:
+                w_ = 1.5 if (p == f and ex in LEX_EXACT[l]) or ascii_ and p in LEX_EXACT[l] else 0.7
+                s[l] += amb * (w_ if l != "pcm" else 1.0)
+    if f in PCM_STRONG and ascii_:                           # Pidgin markers are plain ASCII ("ọmọ" is Yoruba)
+        s["pcm"] += 4
+    # orthography hints (weak)
+    if ascii_:
+        if re.search(r"(th|wh|ck|tion|ing$|ould$|ght)", f):
+            s["eng"] += 1
+        if re.match(r"^(nw|kw|gw|mm|nn|nk|ng[a-z])", f) and f not in LEX["eng"]:
+            s["ibo"] += 0.7
+        if re.search(r"(ts|^ky|^'y|aa|ii|uu)", f) and f not in LEX["eng"]:
+            s["hau"] += 0.5
+    return s, bool(ascii_)
+
+
+def tokenize(text):
+    """-> list of (start, end, word) for words; everything else is glue."""
+    return [(m.start(), m.end(), m.group(0)) for m in WORD.finditer(text)]
+
+
+def clauses(text, toks):
+    """clause index for each word: a new clause starts after , ; : . ? !"""
+    ends = [m.start() for m in CLAUSE_END.finditer(text)]
+    out, k = [], 0
+    for a, b, w in toks:
+        while k < len(ends) and ends[k] < a:
+            k += 1
+        out.append(k)
+    return out
+
+
+def _norm(d):
+    t = sum(max(0.0, v) for v in d.values())
+    return {k: (max(0.0, v) / t if t else 0.0) for k, v in d.items()}
+
+
+class Tagger:
+    def __init__(self, lid=None, lang_map=None):
+        self.lang_map = {fold(k): v for k, v in (lang_map or {}).items()}
+        self.lid = None
+        if lid == "glotlid":
+            import fasttext
+            from huggingface_hub import hf_hub_download
+            self.lid = fasttext.load_model(hf_hub_download("cis-lmu/glotlid", "model.bin"))
+
+    def _lid_scores(self, text):
+        labs, ps = self.lid.predict(text.replace("\n", " "), k=20)
+        m = {"__label__eng_Latn": "eng", "__label__pcm_Latn": "pcm", "__label__ibo_Latn": "ibo",
+             "__label__yor_Latn": "yor", "__label__hau_Latn": "hau"}
+        s = dict.fromkeys(LANGS, 0.0)
+        for l, p in zip(labs, ps):
+            if l in m:
+                s[m[l]] += float(p)
+        return s
+
+    def word_langs(self, text):
+        toks = tokenize(text)
+        if not toks:
+            return toks, []
+        cl = clauses(text, toks)
+        n = len(toks)
+        sc, asc = zip(*[word_scores(w) for _, _, w in toks])
+        sc = [dict(s) for s in sc]
+        f = [fold(w) for _, _, w in toks]
+        # Pidgin bigrams and "pronoun go verb"
+        for i in range(n - 1):
+            if cl[i] != cl[i + 1]:
+                continue
+            if (f[i], f[i + 1]) in PCM_BIGRAMS:
+                sc[i]["pcm"] += 2.5; sc[i + 1]["pcm"] += 2.5
+            if i + 2 < n and f[i] in PRON and f[i + 1] == "go" and cl[i + 2] == cl[i] and f[i + 2] not in NOT_VERB_AFTER_GO \
+                    and f[i + 2] in LEX["eng"] | LEX["pcm"]:
+                for j in (i, i + 1, i + 2):
+                    sc[j]["pcm"] += 2.5
+        # clause-level evidence
+        C = sorted(set(cl))
+        cs = {c: dict.fromkeys(LANGS, 0.0) for c in C}
+        for i in range(n):
+            for l in LANGS:
+                cs[cl[i]][l] += sc[i][l]
+        # clause-final "o" in an ASCII clause = Pidgin particle (Yoruba "o" clauses carry other Yoruba evidence)
+        for i in range(n):
+            last = i == n - 1 or cl[i + 1] != cl[i]
+            if last and f[i] in PCM_AMBIG_END and asc[i]:
+                others = [j for j in range(n) if cl[j] == cl[i] and j != i]
+                if others and all(asc[j] for j in others) and cs[cl[i]]["yor"] < 2:
+                    sc[i]["pcm"] += 3; cs[cl[i]]["pcm"] += 3
+        if self.lid is not None:
+            for c in C:
+                ws = [toks[i][2] for i in range(n) if cl[i] == c]
+                for l, v in self._lid_scores(" ".join(ws)).items():
+                    cs[c][l] += 2.0 * v
+        out = []
+        for i in range(n):
+            c = cl[i]
+            ctx = _norm(cs[c])
+            nb = dict.fromkeys(LANGS, 0.0)        # neighbouring clauses, weaker
+            for d in (-1, 1):
+                if c + d in cs:
+                    for l, v in _norm(cs[c + d]).items():
+                        nb[l] += 0.3 * v
+            tot = {l: sc[i][l] + 1.5 * ctx[l] + nb[l] for l in LANGS}
+            # plain-ASCII words in a clause without Nigerian-language evidence: English or Pidgin
+            if asc[i] and max(sc[i][l] for l in NG) < 1.0:
+                ng_ctx = max(cs[c][l] for l in NG)
+                if ng_ctx < 1.5 * max(cs[c]["eng"], cs[c]["pcm"], 0.1):
+                    pc = cs[c]["pcm"] >= 2.5
+                    out.append("pcm" if pc else ("eng" if sc[i]["pcm"] < 4 else "pcm"))
+                    continue
+            best = max(tot, key=tot.get)
+            if tot[best] <= 0:
+                best = "eng"
+            out.append(best)
+        for i in range(n):                         # user overrides
+            if f[i] in self.lang_map:
+                out[i] = self.lang_map[f[i]]
+        return toks, out
+
+    def tag(self, text, force_lang=None):
+        text = unicodedata.normalize("NFC", text.strip())
+        if TAGGED.search(text):
+            return text                           # already tagged: leave alone
+        if force_lang:
+            return f"<{force_lang}>{text}</{force_lang}>"
+        toks, langs = self.word_langs(text)
+        if not toks:
+            return f"<eng>{text}</eng>"
+        # spans: consecutive same-language words; glue (spaces/punct) stays with the preceding word
+        spans, cur_l, a = [], langs[0], 0
+        for i in range(1, len(toks)):
+            if langs[i] != cur_l:
+                b = toks[i][0]
+                seg = text[a:b]
+                spans.append((cur_l, seg))
+                cur_l, a = langs[i], b
+        spans.append((cur_l, text[a:]))
+        lead = text[:toks[0][0]]
+        out = []
+        for k, (l, s) in enumerate(spans):
+            s2 = (lead + s) if k == 0 else s
+            core = s2.rstrip()
+            out.append(f"<{l}>{core}</{l}>")
+        return " ".join(out)
+
+
+def parse_map(s):
+    m = {}
+    for kv in (s or "").split(","):
+        if "=" in kv:
+            k, v = kv.split("=", 1)
+            v = v.strip()
+            if v not in LANGS:
+                raise ValueError(f"--lang_map: unknown language '{v}' (use {LANGS})")
+            m[k.strip()] = v
+    return m
+
+
+def spans_of(tagged):
+    return [(m.group(1), m.group(2)) for m in re.finditer(r"<(eng|ibo|hau|yor|pcm)>(.*?)</\1>", tagged, re.S)]
+
+
+def word_labels(tagged):
+    """gold/pred word-level labels from a tagged string."""
+    out = []
+    for l, s in spans_of(tagged):
+        out += [(fold(w), l) for _, _, w in tokenize(s)]
+    return out
+
+
+def strip_tags(tagged):
+    return re.sub(r"\s+", " ", re.sub(r"</?(eng|ibo|hau|yor|pcm)>", "", tagged)).strip()
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("text")
+    ap.add_argument("--lang_map", default="", help='per-word overrides, e.g. "omo=pcm,sha=pcm"')
+    ap.add_argument("--force_lang", choices=LANGS)
+    ap.add_argument("--lid", choices=["none", "glotlid"], default="none")
+    a = ap.parse_args()
+    t = Tagger(None if a.lid == "none" else a.lid, parse_map(a.lang_map))
+    print(t.tag(a.text, a.force_lang))
+
+
+if __name__ == "__main__":
+    main()

code/tts_engine.py

@@ -0,0 +1,93 @@
+"""Streaming TTS with Flash-DPO (Dynamic Accent, CosyVoice3 engine): the tagged reply is cut into chunks
+(a short first chunk for latency, then sentence by sentence) and each chunk is synthesised as soon as the previous is done."""
+import os, sys, re, time
+import numpy as np
+REL = os.environ.get("DA_REL", "/home/dev/fi/reltest")
+sys.path.insert(0, REL)
+os.environ.setdefault("CV3_DIR", "/home/dev/models/Fun-CosyVoice3-0.5B-2512")
+os.environ.setdefault("DA_PROMPTS", REL + "/prompts")
+TAG = re.compile(r"<(eng|ibo|yor|hau|pcm)>(.*?)</\1>", re.S)
+VOICE = {"anchor_male": "anchor", "waxal_ibo_5_female": "waxal_ibo_5", "waxal_yor_5_female": "waxal_yor_5"}
+
+def chunk_tagged(tagged, first_max_words=6):
+    """-> list of tagged chunk strings. Splits at sentence ends (. ! ?), keeps tags, first chunk short (also cut at a comma/greeting)."""
+    units, cur = [], []
+    for l, t in TAG.findall(tagged):
+        parts = re.split(r"(?<=[.!?])\s+", t.strip())
+        for k, s in enumerate(parts):
+            if not s: continue
+            cur.append((l, s))
+            if re.search(r"[.!?]$", s):
+                units.append(cur); cur = []
+    if cur: units.append(cur)
+    if not units: return []
+    first = units[0]
+    n = sum(len(s.split()) for _, s in first)
+    if n > first_max_words:                       # fast first chunk: cut the first sentence at a comma / tag boundary / conjunction
+        if len(first) > 1 and len(first[0][1].split()) <= first_max_words:
+            units[0] = [first[0]]; units.insert(1, first[1:])
+        else:
+            l, s = first[0]
+            m = re.search(r"[,;:]\s", s)
+            if not (m and 1 <= len(s[:m.start()].split()) <= first_max_words):
+                m = None
+                ws = s.split()
+                for k in range(3, min(first_max_words, len(ws) - 3)):
+                    if ws[k].lower() in ("and", "but", "so", "because", "then", "or"):
+                        m = re.search(r"\s" + re.escape(ws[k]) + r"\s", s); break
+            if m:
+                cut = m.end() if s[m.start()] in ",;:" else m.start()
+                units[0] = [(l, s[:cut].strip())]; units.insert(1, [(l, s[cut:].strip())] + first[1:])
+    # hard caps: first chunk <= 10 words, later chunks <= 22 words (split at a comma if possible, else at a word boundary)
+    def split_unit(u, cap):
+        out, cur, n = [], [], 0
+        for l, s in u:
+            ws = s.split()
+            while n + len(ws) > cap:
+                k = cap - n
+                sub = " ".join(ws[:k]); c = sub.rfind(",")
+                if c > len(sub) // 3: k = len(sub[:c + 1].split())
+                if k > 0: cur.append((l, " ".join(ws[:k])))
+                out.append(cur); cur, n, ws = [], 0, ws[k:]
+            if ws: cur.append((l, " ".join(ws))); n += len(ws)
+        if cur: out.append(cur)
+        return out
+    capped = []
+    for i, u in enumerate(units):
+        capped += split_unit(u, 10 if i == 0 else 22)
+    units = [u for u in capped if u]
+    return [" ".join(f"<{l}>{s}</{l}>" for l, s in u) for u in units]
+
+class TTS:
+    def __init__(self, voice="anchor_male", lm=None, diac=None, tail_pad=4):
+        import flash_lm
+        from frontend import FrontEnd
+        lm = lm or f"{REL}/checkpoints/flash/flash_dpo/best.pt"
+        diac = diac or f"{REL}/checkpoints/flash/diacritics_byt5"
+        self.S = flash_lm.Synth(lm)
+        self.fe = FrontEnd(diac if os.path.isdir(diac) else None)
+        self.sr = self.S.cosy.sample_rate
+        self.voice = VOICE.get(voice, voice); self.tail_pad = tail_pad
+        self.synth("<eng>Warm up.</eng>")          # compile/cache voice prompt
+
+    def synth(self, tagged, seed=0, voice=None):
+        import torch
+        v = VOICE.get(voice, voice) if voice else self.voice
+        t0 = time.perf_counter()
+        # diacritic restoration only for ibo/yor spans written without any tone marks (the LLM usually writes them)
+        need = any(l in ("ibo", "yor") and not re.search(r"[\u0300-\u036f\u1ea0-\u1ef9ịụọẹṣńḿ]", __import__("unicodedata").normalize("NFD", s) + s) for l, s in TAG.findall(tagged))
+        t = self.fe(tagged, "codeswitch", "eng") if need else tagged
+        t1 = time.perf_counter()
+        toks = list(self.S.tokens(t, v, "normal", seed)[0]); torch.cuda.synchronize()
+        t2 = time.perf_counter()
+        if self.tail_pad: toks += [1] * self.tail_pad
+        w = np.asarray(self.S.wav(toks, v), dtype=np.float32)
+        t3 = time.perf_counter()
+        self.last = dict(fe=round(t1 - t0, 3), lm=round(t2 - t1, 3), flow_voc=round(t3 - t2, 3), n_tok=len(toks), audio_s=round(len(w) / self.sr, 2))
+        return w
+
+    def stream(self, tagged, seed=0, voice=None):
+        """yields (chunk_text, wav, synth_seconds)"""
+        for i, c in enumerate(chunk_tagged(tagged)):
+            t = time.perf_counter(); w = self.synth(c, seed + 101 * i, voice)
+            yield c, w, time.perf_counter() - t

code/turn.py

@@ -0,0 +1,68 @@
+"""Duplex turn-taking: Silero VAD (MIT) per 32 ms frame, end-of-turn by trailing silence, barge-in while the agent
+speaks/thinks, and light backchannel cues at mid-turn pauses. Pure frame clock (16 kHz, 512-sample frames)."""
+import torch, numpy as np
+FR = 512; SR = 16000; FMS = FR / SR
+
+class TurnDetector:
+    def __init__(self, eot_ms=600, start_ms=96, barge_ms=192, on=0.5, off=0.35, preroll_ms=300,
+                 bc_min_turn_s=4.5, bc_pause_ms=(288, 448), bc_gap_s=10.0, spec_ms=256):
+        from silero_vad import load_silero_vad
+        self.vad = load_silero_vad()
+        self.eot_f = int(eot_ms / 1000 / FMS); self.start_f = max(1, int(start_ms / 1000 / FMS)); self.barge_f = int(barge_ms / 1000 / FMS)
+        self.on, self.off = on, off; self.pre_f = int(preroll_ms / 1000 / FMS)
+        self.spec_f = int(spec_ms / 1000 / FMS) if spec_ms else 0
+        self.bc_min, self.bc_pause, self.bc_gap = bc_min_turn_s, (int(bc_pause_ms[0] / 1000 / FMS), int(bc_pause_ms[1] / 1000 / FMS)), bc_gap_s
+        self.reset()
+
+    def reset(self):
+        self.vad.reset_states(); self.t = 0.0; self.in_speech = False; self.on_count = 0; self.sil = 0
+        self.ring = []; self.buf = []; self.speech_frames = 0; self.onset_t = None; self.last_bc = -99; self.bc_done_this_pause = False
+        self.barged = False; self.busy_speech = 0; self._was_busy = False
+
+    def feed(self, frame, agent_busy=False):
+        """frame: float32[512]. returns list of (event, info). events: start, barge, backchannel, eot"""
+        ev = []
+        p = float(self.vad(torch.from_numpy(np.ascontiguousarray(frame, dtype=np.float32)), SR).item())
+        self.t += FMS
+        sp = p > (self.off if self.in_speech else self.on)
+        if agent_busy and not self._was_busy:
+            self.busy_speech = 0; self.barged = False
+        self._was_busy = agent_busy
+        if agent_busy:
+            self.busy_speech = self.busy_speech + 1 if p > self.on else max(0, self.busy_speech - 1)
+            if not self.barged and self.busy_speech >= self.barge_f:
+                self.barged = True; ev.append(("barge", dict(t=self.t, onset=self.onset_t)))
+        if not self.in_speech:
+            self.ring.append(frame); self.ring = self.ring[-self.pre_f:]
+            self.on_count = self.on_count + 1 if sp else 0
+            if self.on_count >= self.start_f:
+                self.in_speech = True; self.sil = 0; self.speech_frames = self.on_count
+                self.onset_t = self.t - self.on_count * FMS; self.buf = list(self.ring); self.bc_done_this_pause = False
+                ev.append(("start", dict(t=self.onset_t, p=p)))
+            if agent_busy and self.on_count >= self.barge_f:
+                pass
+        else:
+            self.buf.append(frame)
+            if sp:
+                if self.spec_f and self.sil >= self.spec_f:
+                    ev.append(("resume", dict(t=self.t)))          # speculative response must be dropped
+                self.sil = 0; self.speech_frames += 1; self.bc_done_this_pause = False
+            else:
+                self.sil += 1
+                if self.spec_f and self.sil == self.spec_f:
+                    keep = len(self.buf) - self.sil + int(0.2 / FMS)
+                    ev.append(("spec", dict(t=self.t, speech_end=self.t - self.sil * FMS, audio=np.concatenate(self.buf[:max(1, keep)]))))
+            dur = self.speech_frames * FMS
+            if (not agent_busy and dur >= self.bc_min and self.bc_pause[0] <= self.sil <= self.bc_pause[1]
+                    and not self.bc_done_this_pause and self.t - self.last_bc >= self.bc_gap):
+                self.bc_done_this_pause = True; self.last_bc = self.t; ev.append(("backchannel", dict(t=self.t, turn_s=dur)))
+            if self.sil >= self.eot_f:
+                end_t = self.t - self.sil * FMS
+                keep = len(self.buf) - self.sil + int(0.2 / FMS)
+                audio = np.concatenate(self.buf[:max(1, keep)])
+                ev.append(("eot", dict(t=self.t, speech_end=end_t, onset=self.onset_t, audio=audio, speech_s=dur)))
+                self.in_speech = False; self.on_count = 0; self.buf = []; self.ring = []
+        return ev
+
+    def partial_audio(self):
+        return np.concatenate(self.buf) if self.buf else np.zeros(0, np.float32)

code/validator.py

@@ -0,0 +1,79 @@
+"""Reply validator: tags well-formed, language proportions vs target mix, greeting pattern kept, length, loops.
+Scores a candidate; the agent keeps the best passing candidate, regenerates up to N rounds, then falls back."""
+import re
+from collections import Counter
+import mix as M
+
+TAG = re.compile(r"<(eng|ibo|yor|hau|pcm)>(.*?)</\1>", re.S)
+
+def parse(out):
+    out = out.strip().splitlines()[0].strip() if out.strip() else ""
+    spans = TAG.findall(out)
+    rest = TAG.sub(" ", out)
+    return out, spans, re.sub(r"[\s.,!?;:'\"\-–—…]", "", rest)
+
+def check(out, target, max_words=40, min_words=2, lid="glotlid", tvd_max=0.3, user=None):
+    out, spans, rest = parse(out)
+    r = dict(text=out, ok=False, why=[], tvd=1.0)
+    if not spans:
+        r["why"].append("no tags"); return r
+    if rest:
+        r["why"].append("text outside tags")
+    words = " ".join(t for _, t in spans).split()
+    if not (min_words <= len(words) <= max_words):
+        r["why"].append(f"length {len(words)}")
+    low = [w.lower().strip(".,!?") for w in words]
+    tri = Counter(tuple(low[i:i + 3]) for i in range(len(low) - 2))
+    if tri and tri.most_common(1)[0][1] > 2:
+        r["why"].append("loop")
+    if user:   # parroting: the LLM copies the user's (often mis-recognised) words instead of answering
+        uw = [w.lower().strip(".,!?") for w in user.split()]
+        ug = {tuple(uw[i:i + 3]) for i in range(len(uw) - 2)}
+        rg = [tuple(low[i:i + 3]) for i in range(len(low) - 2)]
+        r["echo"] = round(sum(g in ug for g in rg) / len(rg), 2) if rg else 0.0
+        if r["echo"] > 0.5 and len(low) >= 6: r["why"].append(f"echo {r['echo']}")   # short closings may legitimately mirror the user
+    # declared mix (by tag) and measured mix (re-tag the plain text with the LID tagger)
+    n = max(1, len(words))
+    decl = Counter()
+    for l, t in spans: decl[l] += len(t.split())
+    decl = {k: v / n for k, v in decl.items()}
+    plain = " ".join(t for _, t in spans)
+    meas = M.estimate(plain, lid=lid)
+    # measured labels can confuse English/Pidgin; merge eng+pcm disagreement softly by averaging declared and measured
+    mixd = {k: 0.5 * decl.get(k, 0) + 0.5 * meas["mix"].get(k, 0) for k in set(decl) | set(meas["mix"])}
+    tgt = target["mix"]
+    d = M.tvd(mixd, tgt)
+    r.update(declared={k: round(v, 2) for k, v in decl.items()}, measured=meas["mix"], mix=({k: round(v, 2) for k, v in mixd.items() if v > 0.01}), tvd=round(d, 3))
+    # tag honesty: words tagged ibo/yor/hau that the tagger thinks are English
+    lab = meas["labels"]; i = 0; mism = 0; ng = 0
+    for l, t in spans:
+        k = len(M.tagger(lid).word_langs(t)[1]) if t.strip() else 0
+        if l in ("ibo", "yor", "hau"):
+            ng += k; mism += sum(1 for x in lab[i:i + k] if x == "eng")
+        i += k
+    r["tag_mismatch"] = round(mism / ng, 2) if ng else 0.0
+    if r["tag_mismatch"] > 0.5: r["why"].append("non-English tags over English words")
+    if d > tvd_max: r["why"].append(f"mix drift tvd={d:.2f}")
+    for l, p in tgt.items():                     # every substantial target language must appear
+        if p >= 0.15 and mixd.get(l, 0) < 0.05: r["why"].append(f"missing {l}")
+    for p in target.get("pattern", []):          # greeting pattern preserved
+        k, _, l = p.partition(":")
+        if k == "greeting" and spans and spans[0][0] != l: r["why"].append(f"no {l} greeting")
+    r["ok"] = not r["why"]
+    r["score"] = d + 0.25 * len(r["why"]) + 0.2 * r["tag_mismatch"]
+    return r
+
+SAFE = {"ibo": "Ndewo!", "yor": "Ẹ pẹ̀lẹ́ o!", "hau": "Sannu!", "pcm": "Abeg no vex,"}
+
+def fallback(text_candidates, target):
+    """all rounds failed. Echoing/drifting native-language output -> an honest short reply (a correct phrasebook opener in the user's
+    language + a clarification request); otherwise the best-scoring candidate if its tags parse; else plain text in the matrix language."""
+    best = min(text_candidates, key=lambda c: c["score"] if "score" in c else 9, default=None)
+    m = target.get("matrix", "eng")
+    if m in SAFE and (best is None or any(w.startswith(("echo", "mix drift", "missing", "no tags", "non-English")) for w in best.get("why", []))):
+        tail = "<pcm>I no too hear you well. Abeg talk am again?</pcm>" if m == "pcm" else "<eng>Sorry, I did not catch all of that. Could you say it again, or in English?</eng>"
+        return f"<{m}>{SAFE[m]}</{m}> {tail}", "safe_phrasebook"
+    if best and TAG.search(best["text"] or ""):
+        return best["text"], "best_failed"
+    m = target.get("matrix", "eng"); raw = re.sub(r"</?\w+>", "", (best or {}).get("text", "") or "Sorry, say that again?")
+    return f"<{m}>{raw.strip()}</{m}>", "plain_matrix"

config.json

@@ -0,0 +1,98 @@
+{
+  "name": "dynamic-accent-duplex",
+  "version": "0.2.0-experimental",
+  "type": "cascaded full-duplex speech-to-speech voice agent (pipeline, not a single model)",
+  "languages": [
+    "en",
+    "pcm",
+    "ig",
+    "yo",
+    "ha"
+  ],
+  "code_license": "apache-2.0",
+  "library_name": "pytorch",
+  "components": {
+    "vad": {
+      "model": "snakers4/silero-vad (v5, pip silero-vad)",
+      "license": "MIT",
+      "settings": {
+        "frame_ms": 32,
+        "speech_start_ms": 96,
+        "end_of_turn_silence_ms": 600,
+        "speculative_start_ms": 256,
+        "barge_in_speech_ms": 192
+      }
+    },
+    "asr": {
+      "model": "Dynamic ASR real (Modularcomputing/dynamic-asr, real/model.pt) = Meta omniASR_CTC_1B_v2 fine-tuned on licensed Nigerian speech",
+      "source": "https://huggingface.co/Modularcomputing/dynamic-asr",
+      "license": "CC-BY-SA-4.0 (weights; base Apache-2.0)",
+      "alternatives": {
+        "dynamic-asr-cs": "codeswitch-experimental/model.pt (experimental)",
+        "omni": "facebook omniASR_CTC_1B (Apache-2.0), also the automatic fallback"
+      },
+      "env": "ASR_MODEL=dynamic-asr|dynamic-asr-cs|omni",
+      "settings": {
+        "decoding": "greedy CTC, language-agnostic, Latin-script logit mask",
+        "partials_every_s": 0.64
+      }
+    },
+    "language_id": {
+      "model": "Dynamic Accent tagger.py (rules: diacritics, lexicons, Pidgin markers) + GlotLID v3 sliding windows",
+      "glotlid": "cis-lmu/glotlid",
+      "license": "Apache-2.0 (tagger code) / Apache-2.0 (GlotLID)"
+    },
+    "reply_llm": {
+      "model": "Qwen/Qwen3-30B-A3B-Instruct-2507-FP8",
+      "license": "Apache-2.0",
+      "server": "vLLM (Apache-2.0), OpenAI-compatible, served name 'llm'",
+      "settings": {
+        "candidates_per_round": 3,
+        "rounds": 2,
+        "temperature": [
+          0.7,
+          0.9
+        ],
+        "max_tokens": 110
+      }
+    },
+    "validator": {
+      "checks": [
+        "tag format",
+        "mix TVD <= 0.3 vs target",
+        "every target language >= 15% present",
+        "greeting pattern kept",
+        "length 2-40 words",
+        "no loops",
+        "no echo of the user's words",
+        "non-English tags not over English words"
+      ],
+      "fallback": "best candidate if tags parse; for ibo/yor/hau/pcm-matrix turns that echo or drift: a phrasebook opener in the user's language + a clarification request"
+    },
+    "tts": {
+      "model": "Dynamic Accent Flash-DPO (checkpoints/flash/flash_dpo in Modularcomputing/dynamic-accent-pro-v1.1)",
+      "base": "FunAudioLLM/Fun-CosyVoice3-0.5B-2512 (Apache-2.0)",
+      "license": "CC-BY-SA-4.0 (weights, voice prompts)",
+      "voice": "anchor_male",
+      "streaming": "chunk-level: first chunk <= 6-10 words, then sentences <= 22 words",
+      "diacritics": "Dynamic Accent ByT5 diacritic restorer (CC-BY-SA-4.0; google/byt5-small Apache-2.0 base), only for ibo/yor spans without tone marks"
+    },
+    "backchannels": {
+      "phrases": {
+        "eng": "Mm-hmm.",
+        "pcm": "Ehen.",
+        "ibo": "Ehen.",
+        "yor": "Ehen.",
+        "hau": "To."
+      },
+      "rule": "user turn >= 4.5 s, mid-turn pause 288-448 ms, >= 10 s apart"
+    }
+  },
+  "not_included_weights": "This repo ships code and config only; every model is downloaded from its own repository under its own licence.",
+  "excluded_on_licence": [
+    "African Voices / WaZoBia (TTS use banned)",
+    "NaijaVoices (non-commercial)",
+    "N-ATLaS (1,000-user cap, share-alike)",
+    "Gemma 3 (gated; not tested)"
+  ]
+}

launch.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+# Start the three processes of Dynamic Accent Duplex on one GPU (tested on 1x H100 80GB).
+#  1) vLLM OpenAI server with the reply LLM (Qwen3-30B-A3B-Instruct-2507-FP8, Apache-2.0)   :8001
+#  2) ASR worker: Dynamic ASR (Modularcomputing/dynamic-asr, fine-tuned omniASR_CTC_1B_v2; needs `pip install "omnilingual-asr>=0.2.0" huggingface_hub`) :8002
+#     ASR_MODEL=dynamic-asr (default) | dynamic-asr-cs | omni (stock omniASR_CTC_1B; also the automatic fallback)
+#  3) duplex server (VAD + turn-taking + LID/mix + validator + Flash TTS) + web UI             :7860
+# TTS: set DA_REL to a local copy of huggingface.co/Modularcomputing/dynamic-accent-pro-v1.1 (Flash-DPO weights,
+# CC-BY-SA-4.0) set up as its README describes (CosyVoice repo + Fun-CosyVoice3-0.5B-2512 base, CV3_DIR).
+set -e
+HERE=$(cd $(dirname $0); pwd)
+: "${DA_REL:?set DA_REL to the dynamic-accent-pro-v1.1 folder}"
+VLLM=${VLLM:-vllm}; PY_ASR=${PY_ASR:-python}; PY=${PY:-python}
+$VLLM serve Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --served-model-name llm --port 8001 --gpu-memory-utilization 0.62 --max-model-len 8192 &
+(cd $HERE/code && ASR_MODEL=${ASR_MODEL:-dynamic-asr} $PY_ASR -m uvicorn asr_server:app --host 127.0.0.1 --port 8002) &
+until curl -s localhost:8001/v1/models >/dev/null && curl -s localhost:8002/health >/dev/null; do sleep 5; done
+cd $HERE/code && DA_REL=$DA_REL $PY server.py      # open http://localhost:7860 (use headphones)

requirements.txt

@@ -0,0 +1,13 @@
+# duplex server / offline driver (same env as the Dynamic Accent Flash TTS: torch, CosyVoice deps)
+fastapi
+uvicorn[standard]
+websockets
+httpx
+silero-vad          # MIT
+fasttext-wheel      # GlotLID (cis-lmu/glotlid, Apache-2.0) for language ID
+huggingface_hub
+soundfile
+librosa
+numpy
+# ASR worker env (separate venv is fine): omnilingual-asr>=0.2.0 (Apache-2.0), huggingface_hub, fastapi, uvicorn  (Dynamic ASR weights: Modularcomputing/dynamic-asr)
+# LLM server: vllm

samples/01_igbo_greetings_english.json

@@ -0,0 +1,354 @@
+{
+ "name": "01_igbo_greetings_english",
+ "desc": "Mostly English with Igbo greetings/closing (ACE's target case). Includes a long turn (backchannel) and a barge-in. User turns are TTS-synthesized (Flash, female Igbo voice).",
+ "user_turns": [
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<ibo>Ndewo!</ibo> <eng>Good morning. Please, can you help me plan my day? I have a lot to do.</eng>"
+  },
+  {
+   "label": "TTS-synthesized user turn (long, mid-turn pause)",
+   "voice": "waxal_ibo_5_female",
+   "text_parts": [
+    "<eng>I need to go to the bank, and then the market,</eng>",
+    "<eng>and in the evening I also want to call my mother.</eng>"
+   ],
+   "inner_pause": 0.4
+  },
+  {
+   "label": "TTS-synthesized user turn (barge-in 1.5 s into the reply)",
+   "voice": "waxal_ibo_5_female",
+   "text": "<eng>Wait, sorry, the bank closes early today.</eng>",
+   "barge_at": 1.5
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<eng>Okay, thank you so much.</eng> <ibo>Daalụ!</ibo>"
+  }
+ ],
+ "turns": [
+  {
+   "turn": 0,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<ibo>Ndewo!</ibo> <eng>Good morning. Please, can you help me plan my day? I have a lot to do.</eng>",
+   "user_start": 0.7,
+   "user_onset": 0.762,
+   "backchannels": [],
+   "asr": "nde wo good morning please can you help me plan my day i have a lot to do",
+   "asr_s": 0.073,
+   "eot_wait_s": 0.576,
+   "speech_end": 5.984,
+   "target": {
+    "mix": {
+     "eng": 0.89,
+     "ibo": 0.11
+    },
+    "matrix": "eng",
+    "pattern": [
+     "greeting:ibo",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: English about 90%, Igbo about 10% of the words. The main (matrix) language is English. The user greeted in Igbo: open with a short Igbo greeting.",
+   "reply": "<ibo>Ndewo!</ibo> <eng>Of course! Let’s start with your most urgent tasks. What’s the first thing you need to do today?</eng>",
+   "reply_mix": {
+    "eng": 0.95,
+    "ibo": 0.05
+   },
+   "tvd": 0.059,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.216,
+     "cands": [
+      {
+       "text": "<ibo>Ndewo!</ibo> <eng>Of course! Let’s start with your most urgent tasks. What’s the first thing you need to do today?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.059
+      },
+      {
+       "text": "<ibo>Ndewo!</ibo> <eng>Sure, I can help. Let’s list your tasks first—what’s the most urgent thing you need to do today?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.059
+      },
+      {
+       "text": "<ibo>Ndewo!</ibo> <eng>Of course! Let’s start with your most urgent tasks. What’s the first thing you need to do today?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.059
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0124,
+   "llm_s": 0.216,
+   "tts_first_s": 0.781,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.096,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.003,
+    "flow_voc": 0.214,
+    "n_tok": 60,
+    "audio_s": 2.4
+   },
+   "tts_chunks_s": [
+    0.781,
+    0.69,
+    1.495,
+    1.217
+   ],
+   "first_audio": 7.335,
+   "reply_end": 15.295,
+   "latency_s": 1.351
+  },
+  {
+   "turn": 1,
+   "user_label": "TTS-synthesized user turn (long, mid-turn pause)",
+   "user_ref": "",
+   "user_start": 15.995,
+   "user_onset": 16.051,
+   "backchannels": [
+    {
+     "t": 22.304,
+     "turn_s": 5.28
+    }
+   ],
+   "backchannel_at": 22.304,
+   "asr": "i need to go to the bank and then the market and in the evening i also want to call my mother",
+   "asr_s": 0.065,
+   "eot_wait_s": 0.576,
+   "speech_end": 22.016,
+   "target": {
+    "mix": {
+     "eng": 0.97,
+     "ibo": 0.03
+    },
+    "matrix": "eng",
+    "pattern": []
+   },
+   "target_desc": "Match the user's language mix: English about 95%, Igbo about 5% of the words. The main (matrix) language is English.",
+   "reply": "<eng>Okay, I got you. Start with the bank, then the market, and save calling your mother for evening. Want me to set a reminder?</eng>",
+   "reply_mix": {
+    "eng": 1.0
+   },
+   "tvd": 0.03,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.257,
+     "cands": [
+      {
+       "text": "<eng>Okay, I’ll help you plan. Start with the bank—go early to avoid queues. Then the market. Call your mother after dinner, when you’re relaxed.</eng> <ibo>Ọ dị mma.</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.081
+      },
+      {
+       "text": "<eng>Okay, I got you. Start with the bank, then the market, and save calling your mother for evening. Want me to set a reminder?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.03
+      },
+      {
+       "text": "<eng>Okay, let’s plan that. Go to the bank first, then the market, and call your mum in the evening. Need a reminder for the call?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.03
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0158,
+   "llm_s": 0.257,
+   "tts_first_s": 0.976,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.333,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 0.966,
+    "flow_voc": 0.217,
+    "n_tok": 58,
+    "audio_s": 2.32
+   },
+   "tts_chunks_s": [
+    0.976,
+    2.443,
+    1.183
+   ],
+   "first_audio": 23.605,
+   "reply_end": 33.528,
+   "latency_s": 1.589
+  },
+  {
+   "turn": 2,
+   "user_label": "TTS-synthesized user turn (barge-in 1.5 s into the reply)",
+   "user_ref": "<eng>Wait, sorry, the bank closes early today.</eng>",
+   "user_start": 25.105,
+   "user_onset": 25.174,
+   "backchannels": [],
+   "barge": {
+    "user_onset": 25.174,
+    "detected": 25.376,
+    "agent_stopped": 25.416,
+    "reaction_s": 0.242,
+    "agent_chunks_played": 1,
+    "agent_chunks_total": 3
+   },
+   "asr": "wait sorry the bank closes ale today",
+   "asr_s": 0.057,
+   "eot_wait_s": 0.576,
+   "speech_end": 27.488,
+   "target": {
+    "mix": {
+     "eng": 0.98
+    },
+    "matrix": "eng",
+    "pattern": []
+   },
+   "target_desc": "Match the user's language mix: English about 100% of the words. The main (matrix) language is English. Reply only in English.",
+   "reply": "<eng>Okay, no problem. Let’s plan for tomorrow instead. Should we adjust your market visit or call time?</eng>",
+   "reply_mix": {
+    "eng": 1.0
+   },
+   "tvd": 0.01,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.246,
+     "cands": [
+      {
+       "text": "<eng>Okay, no problem. Let’s plan for tomorrow instead. Should we adjust your market visit or call time?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.01
+      },
+      {
+       "text": "<eng>Okay, no problem. Let’s adjust—go to the market first, then call your mum in the evening. Want me to set a reminder?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.01
+      },
+      {
+       "text": "<eng>Oh no worries—let’s adjust. We can go to the market first, then call your mum, and the bank can wait till tomorrow.</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.01
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0055,
+   "llm_s": 0.246,
+   "tts_first_s": 0.891,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.214,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.424,
+    "flow_voc": 0.221,
+    "n_tok": 85,
+    "audio_s": 3.4
+   },
+   "tts_chunks_s": [
+    0.891,
+    1.124,
+    1.645
+   ],
+   "first_audio": 28.958,
+   "reply_end": 36.358,
+   "latency_s": 1.47
+  },
+  {
+   "turn": 3,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<eng>Okay, thank you so much.</eng> <ibo>Daalụ!</ibo>",
+   "user_start": 37.058,
+   "user_onset": 37.119,
+   "backchannels": [],
+   "asr": "okay thank you somuch daalo",
+   "asr_s": 0.056,
+   "eot_wait_s": 0.576,
+   "speech_end": 39.488,
+   "target": {
+    "mix": {
+     "eng": 0.79,
+     "ibo": 0.2
+    },
+    "matrix": "eng",
+    "pattern": [
+     "closing:ibo",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: English about 80%, Igbo about 20% of the words. The main (matrix) language is English. The user closed in Igbo: you may close in Igbo.",
+   "reply": "<eng>My pleasure!</eng> <ibo>Daalụ!</ibo>",
+   "reply_mix": {
+    "eng": 0.67,
+    "ibo": 0.33
+   },
+   "tvd": 0.127,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.177,
+     "cands": [
+      {
+       "text": "<eng>My pleasure!</eng> <ibo>Daalụ!</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.127
+      },
+      {
+       "text": "<eng>Anytime, no wahala.</eng> <ibo>Daalụ ọma!</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.49"
+       ],
+       "tvd": 0.495
+      },
+      {
+       "text": "<eng>Anytime, no wahala.</eng> <ibo>Daalụ!</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.42"
+       ],
+       "tvd": 0.42
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0042,
+   "llm_s": 0.177,
+   "tts_first_s": 0.758,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.002,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 0.46,
+    "flow_voc": 0.203,
+    "n_tok": 30,
+    "audio_s": 1.2
+   },
+   "tts_chunks_s": [
+    0.758,
+    0.662
+   ],
+   "first_audio": 40.746,
+   "reply_end": 43.346,
+   "latency_s": 1.258
+  }
+ ]
+}

samples/01_igbo_greetings_english.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18f762b80bc3ab09b920201869b0a8c4070a80cb6323ad2d1be38b92ce03d9f4
+size 704684

samples/03_pidgin_bargein.json

@@ -0,0 +1,257 @@
+{
+ "name": "03_pidgin_bargein",
+ "desc": "Nigerian Pidgin throughout, with a barge-in. User turns TTS-synthesized (Flash, female voice).",
+ "user_turns": [
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_yor_5_female",
+   "text": "<pcm>How far? Abeg, wetin I fit cook tonight wey no go take long?</pcm>"
+  },
+  {
+   "label": "TTS-synthesized user turn (barge-in 1.2 s into the reply)",
+   "voice": "waxal_yor_5_female",
+   "text": "<pcm>No be rice o, I no get rice for house.</pcm>",
+   "barge_at": 1.2
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_yor_5_female",
+   "text": "<pcm>Okay, na so. Thank you well well.</pcm>"
+  }
+ ],
+ "turns": [
+  {
+   "turn": 0,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<pcm>How far? Abeg, wetin I fit cook tonight wey no go take long?</pcm>",
+   "user_start": 0.7,
+   "user_onset": 0.764,
+   "backchannels": [],
+   "asr": "how far i beg waiting i fik cook tonight when wou go take long",
+   "asr_s": 0.069,
+   "eot_wait_s": 0.576,
+   "speech_end": 4.768,
+   "target": {
+    "mix": {
+     "pcm": 1.0
+    },
+    "matrix": "pcm",
+    "pattern": [
+     "greeting:pcm"
+    ]
+   },
+   "target_desc": "Match the user's language mix: Nigerian Pidgin about 100% of the words. The main (matrix) language is Nigerian Pidgin. The user greeted in Nigerian Pidgin: open with a short Nigerian Pidgin greeting. Reply only in Nigerian Pidgin.",
+   "reply": "<pcm>How far? Abeg, jollof rice with pepper soup no go take long, only twenty minutes. You fit start now?</pcm>",
+   "reply_mix": {
+    "pcm": 1.0
+   },
+   "tvd": 0.0,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.23,
+     "cands": [
+      {
+       "text": "<pcm>How far? Abeg, jollof rice with pepper soup no go take long, only twenty minutes. You fit start now?</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>How far! Abeg, jollof rice with beans no go take more than thirty minutes. You fit start now?</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>How far! Abeg, jollof rice with chicken no go take long, less than thirty minutes. You fit start now?</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0106,
+   "llm_s": 0.23,
+   "tts_first_s": 0.795,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.119,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 0.592,
+    "flow_voc": 0.216,
+    "n_tok": 38,
+    "audio_s": 1.52
+   },
+   "tts_chunks_s": [
+    0.795,
+    2.575,
+    0.808
+   ],
+   "first_audio": 6.143,
+   "reply_end": 15.678,
+   "latency_s": 1.375
+  },
+  {
+   "turn": 1,
+   "user_label": "TTS-synthesized user turn (barge-in 1.2 s into the reply)",
+   "user_ref": "<pcm>No be rice o, I no get rice for house.</pcm>",
+   "user_start": 7.343,
+   "user_onset": 7.609,
+   "backchannels": [],
+   "barge": {
+    "user_onset": 7.609,
+    "detected": 7.808,
+    "agent_stopped": 7.848,
+    "reaction_s": 0.239,
+    "agent_chunks_played": 1,
+    "agent_chunks_total": 3
+   },
+   "asr": "no be rice o i no get rice for huse",
+   "asr_s": 0.062,
+   "eot_wait_s": 0.576,
+   "speech_end": 11.2,
+   "target": {
+    "mix": {
+     "pcm": 1.0
+    },
+    "matrix": "pcm",
+    "pattern": [
+     "discourse:pcm"
+    ]
+   },
+   "target_desc": "Match the user's language mix: Nigerian Pidgin about 100% of the words. The main (matrix) language is Nigerian Pidgin. Sprinkle a Nigerian Pidgin discourse word or two the way the user did. Reply only in Nigerian Pidgin.",
+   "reply": "<pcm>No wahala, I dey get rice for house. You fit cook e now, no go take long.</pcm>",
+   "reply_mix": {
+    "pcm": 1.0
+   },
+   "tvd": 0.0,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.214,
+     "cands": [
+      {
+       "text": "<pcm>No wahala, I dey get rice for house. You fit cook e now, no go take long.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>No wahala, I dey get rice for house. You just tell me wetin you fit cook, I go help you quick.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>No wahala, I dey get rice for house. You fit cook e now, no go take long.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0084,
+   "llm_s": 0.214,
+   "tts_first_s": 0.784,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.081,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.385,
+    "flow_voc": 0.208,
+    "n_tok": 85,
+    "audio_s": 3.4
+   },
+   "tts_chunks_s": [
+    0.784,
+    1.183,
+    1.594
+   ],
+   "first_audio": 12.538,
+   "reply_end": 19.738,
+   "latency_s": 1.338
+  },
+  {
+   "turn": 2,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<pcm>Okay, na so. Thank you well well.</pcm>",
+   "user_start": 20.438,
+   "user_onset": 20.708,
+   "backchannels": [],
+   "asr": "ok na so thank you wew",
+   "asr_s": 0.049,
+   "eot_wait_s": 0.576,
+   "speech_end": 24.448,
+   "target": {
+    "mix": {
+     "pcm": 1.0
+    },
+    "matrix": "pcm",
+    "pattern": [
+     "discourse:pcm"
+    ]
+   },
+   "target_desc": "Match the user's language mix: Nigerian Pidgin about 100% of the words. The main (matrix) language is Nigerian Pidgin. Sprinkle a Nigerian Pidgin discourse word or two the way the user did. Reply only in Nigerian Pidgin.",
+   "reply": "<pcm>Ok na so, thank you well well. We go see later, no get problem.</pcm>",
+   "reply_mix": {
+    "pcm": 1.0
+   },
+   "tvd": 0.0,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.185,
+     "cands": [
+      {
+       "text": "<pcm>Ok na so, thank you well well. We go see later, no get problem.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>Ok na so, thank you well well. We go see later, no wahala.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      },
+      {
+       "text": "<pcm>Ok na so, thank you well well. We go see later, no wahala.</pcm>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.0
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0048,
+   "llm_s": 0.185,
+   "tts_first_s": 0.75,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.001,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.034,
+    "flow_voc": 0.219,
+    "n_tok": 64,
+    "audio_s": 2.56
+   },
+   "tts_chunks_s": [
+    0.75,
+    1.156,
+    1.253
+   ],
+   "first_audio": 25.704,
+   "reply_end": 32.024,
+   "latency_s": 1.256
+  }
+ ]
+}

samples/03_pidgin_bargein.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a6c93794a299b388091c0ece2178232ba01088f499628f2b34cc259edfdb94d
+size 523436

samples/04_hausa_greetings_english.json

@@ -0,0 +1,296 @@
+{
+ "name": "04_hausa_greetings_english",
+ "desc": "Hausa greeting + English, Hausa closing. User turns TTS-synthesized (Flash, female voice).",
+ "user_turns": [
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<hau>Sannu, barka da yamma.</hau> <eng>I am travelling to Kano tomorrow.</eng>"
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<eng>What should I pack for the weather there?</eng>"
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<hau>To, na gode.</hau>"
+  }
+ ],
+ "turns": [
+  {
+   "turn": 0,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<hau>Sannu, barka da yamma.</hau> <eng>I am travelling to Kano tomorrow.</eng>",
+   "user_start": 0.7,
+   "user_onset": 0.769,
+   "backchannels": [],
+   "asr": "sannu baka da yamma i am travelling to cano tomorrow",
+   "asr_s": 0.058,
+   "eot_wait_s": 0.576,
+   "speech_end": 5.248,
+   "target": {
+    "mix": {
+     "eng": 0.6,
+     "hau": 0.4
+    },
+    "matrix": "eng",
+    "pattern": [
+     "greeting:hau",
+     "discourse:hau",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: English about 60%, Hausa about 40% of the words. The main (matrix) language is English. The user greeted in Hausa: open with a short Hausa greeting. Sprinkle a Hausa discourse word or two the way the user did.",
+   "reply": "<hau>Sannu!</hau> <eng>Good evening, I hear you're travelling to Kano tomorrow. Is it for business or family?</eng> <hau>Ba mu aji kama?</hau>",
+   "reply_mix": {
+    "eng": 0.75,
+    "hau": 0.25
+   },
+   "tvd": 0.15,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.298,
+     "cands": [
+      {
+       "text": "<hau>Sannu!</hau> <eng>Good evening, I hear you're travelling to Kano tomorrow. Is it for business or family?</eng> <hau>Ba mu aji kama?</hau>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.15
+      },
+      {
+       "text": "<hau>Sannu!</hau> <eng>Good evening. I see you’re travelling to Kano tomorrow. Is it for business or pleasure?</eng>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.31"
+       ],
+       "tvd": 0.309
+      },
+      {
+       "text": "<hau>Sannu baka da yamma!</hau> <eng>That’s good to hear. Are you sure you’ve packed everything for your trip to Kano?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.19
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0077,
+   "llm_s": 0.298,
+   "tts_first_s": 0.668,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.046,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 0.659,
+    "flow_voc": 0.201,
+    "n_tok": 42,
+    "audio_s": 1.68
+   },
+   "tts_chunks_s": [
+    0.668,
+    1.721,
+    1.108,
+    0.86
+   ],
+   "first_audio": 6.55,
+   "reply_end": 15.75,
+   "latency_s": 1.302
+  },
+  {
+   "turn": 1,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<eng>What should I pack for the weather there?</eng>",
+   "user_start": 16.45,
+   "user_onset": 16.695,
+   "backchannels": [],
+   "asr": "what should i park for the weather there",
+   "asr_s": 0.049,
+   "eot_wait_s": 0.576,
+   "speech_end": 18.368,
+   "target": {
+    "mix": {
+     "eng": 0.88,
+     "hau": 0.12
+    },
+    "matrix": "eng",
+    "pattern": []
+   },
+   "target_desc": "Match the user's language mix: English about 90%, Hausa about 10% of the words. The main (matrix) language is English.",
+   "reply": "<eng>For Kano tomorrow, pack light clothes and a light jacket—weather will be warm during the day, cool at night. You need a hat too?</eng> <hau>Ya, kaiya da kaiyawa</hau>",
+   "reply_mix": {
+    "eng": 0.86,
+    "hau": 0.14
+   },
+   "tvd": 0.021,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.335,
+     "cands": [
+      {
+       "text": "<eng>For Kano tomorrow, pack light clothes and a light jacket—weather will be hot during the day, cooler in the evening. You need a hat too?</eng> <hau>Wààtùn kààlà</hau>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.083
+      },
+      {
+       "text": "<eng>For the weather in Kano, pack light clothes and a hat — it’s usually hot and sunny. But bring a light jacket for evenings. You need a raincoat?</eng> <hau>Na gida?</hau>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.052
+      },
+      {
+       "text": "<eng>For Kano tomorrow, pack light clothes and a light jacket—weather will be warm during the day, cool at night. You need a hat too?</eng> <hau>Ya, kaiya da kaiyawa</hau>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.021
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0062,
+   "llm_s": 0.335,
+   "tts_first_s": 0.844,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.253,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.232,
+    "flow_voc": 0.203,
+    "n_tok": 76,
+    "audio_s": 3.04
+   },
+   "tts_chunks_s": [
+    0.844,
+    2.871,
+    0.917,
+    1.435
+   ],
+   "first_audio": 19.876,
+   "reply_end": 33.707,
+   "latency_s": 1.508
+  },
+  {
+   "turn": 2,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<hau>To, na gode.</hau>",
+   "user_start": 34.407,
+   "user_onset": 34.475,
+   "backchannels": [],
+   "asr": "tu na godi",
+   "asr_s": 0.049,
+   "eot_wait_s": 0.576,
+   "speech_end": 35.616,
+   "target": {
+    "mix": {
+     "hau": 0.56,
+     "eng": 0.24,
+     "ibo": 0.2
+    },
+    "matrix": "hau",
+    "pattern": [
+     "closing:hau",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: Hausa about 55%, English about 25%, Igbo about 20% of the words. The main (matrix) language is Hausa. The user closed in Hausa: you may close in Hausa.",
+   "reply": "<hau>Sannu!</hau> <eng>Sorry, I did not catch all of that. Could you say it again, or in English?</eng>",
+   "reply_mix": {
+    "eng": 0.91,
+    "hau": 0.09
+   },
+   "tvd": 0.671,
+   "how": "safe_phrasebook",
+   "first_cand_ok": false,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.264,
+     "cands": [
+      {
+       "text": "<hau>Tu na godi!</hau> <eng>Thank you for the info. Is there anything else I can help with?</eng> <ibo>Daalụ!</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.52"
+       ],
+       "tvd": 0.522
+      },
+      {
+       "text": "<hau>Tu na godi!</hau> <eng>Thank you, I’m glad I could help.</eng> <ibo>Daalụ!</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.42"
+       ],
+       "tvd": 0.424
+      },
+      {
+       "text": "<hau>Tu na Godi!</hau> <eng>Thank you, I’ll keep that in mind.</eng> <ibo>Daalụ ọma!</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.44"
+       ],
+       "tvd": 0.435
+      }
+     ]
+    },
+    {
+     "round": 1,
+     "llm_s": 0.273,
+     "cands": [
+      {
+       "text": "<hau>Tu na Godi!</hau> <eng>Thank you for the help. I’ll make sure to pack for the weather.</eng> <ibo>Dalụ n’ụzọ ịmụ ọrụ.</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.49"
+       ],
+       "tvd": 0.485
+      },
+      {
+       "text": "<hau>Tu na godi!</hau> <eng>Thank you for the info. Is the weather there usually dry this time of year?</eng> <ibo>Ọ dị mma, nke a banyere ike</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.50"
+       ],
+       "tvd": 0.5
+      },
+      {
+       "text": "<hau>Tu na godi!</hau> <eng>Thank you for the info. I’ll keep you posted on the weather.</eng> <ibo>Dalụ na ọma</ibo>",
+       "ok": false,
+       "why": [
+        "mix drift tvd=0.48"
+       ],
+       "tvd": 0.477
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0031,
+   "llm_s": 0.537,
+   "tts_first_s": 0.629,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.249,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.4,
+    "flow_voc": 0.224,
+    "n_tok": 86,
+    "audio_s": 3.44
+   },
+   "tts_chunks_s": [
+    0.629,
+    1.235,
+    1.625
+   ],
+   "first_audio": 37.121,
+   "reply_end": 44.356,
+   "latency_s": 1.505
+  }
+ ]
+}

samples/04_hausa_greetings_english.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c26529b1c1611530b760d00731d371613ec639815d8416a8ec0ef471d12bd622
+size 720812

samples/05_igbo_heavy.json

@@ -0,0 +1,259 @@
+{
+ "name": "05_igbo_heavy",
+ "desc": "Igbo-heavier user (shows the weak spot: LLM Igbo beyond phrases). User turns TTS-synthesized (Flash, female Igbo voice).",
+ "user_turns": [
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<ibo>Kedu? Biko, nyere m aka.</ibo> <eng>I need a taxi to the airport.</eng>"
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<ibo>Ee, ọ dị mma.</ibo> <eng>I will leave at six.</eng>"
+  },
+  {
+   "label": "TTS-synthesized user turn",
+   "voice": "waxal_ibo_5_female",
+   "text": "<ibo>Daalụ, ka ọ dị.</ibo>"
+  }
+ ],
+ "turns": [
+  {
+   "turn": 0,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<ibo>Kedu? Biko, nyere m aka.</ibo> <eng>I need a taxi to the airport.</eng>",
+   "user_start": 0.7,
+   "user_onset": 0.753,
+   "backchannels": [],
+   "asr": "kedu biko nere maaka i need a taxi to the airport",
+   "asr_s": 0.065,
+   "eot_wait_s": 0.576,
+   "speech_end": 5.632,
+   "target": {
+    "mix": {
+     "eng": 0.82,
+     "ibo": 0.18
+    },
+    "matrix": "eng",
+    "pattern": [
+     "greeting:ibo",
+     "discourse:ibo",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: English about 80%, Igbo about 20% of the words. The main (matrix) language is English. The user greeted in Igbo: open with a short Igbo greeting. Sprinkle a Igbo discourse word or two the way the user did.",
+   "reply": "<ibo>Kedu?</ibo> <eng>Okay, I can help you book a taxi to the airport. How soon do you need it?</eng> <ibo>Biko</ibo>",
+   "reply_mix": {
+    "eng": 0.89,
+    "ibo": 0.11
+   },
+   "tvd": 0.072,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.268,
+     "cands": [
+      {
+       "text": "<ibo>Kedu?</ibo> <eng>Okay, I can help you book a taxi to the airport. How soon do you need it?</eng> <ibo>Biko</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.072
+      },
+      {
+       "text": "<ibo>Kedu?</ibo> <eng>Okay, I can help you book a taxi to the airport. How soon do you need it?</eng> <ibo>Biko,</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.072
+      },
+      {
+       "text": "<ibo>Kedu?</ibo> <eng>Okay, I can help you book a taxi to the airport. How far you want the pickup time?</eng> <ibo>Biko</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.175
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0083,
+   "llm_s": 0.268,
+   "tts_first_s": 0.664,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.02,
+   "tts_breakdown": {
+    "fe": 0.035,
+    "lm": 0.088,
+    "flow_voc": 0.198,
+    "n_tok": 8,
+    "audio_s": 0.32
+   },
+   "tts_chunks_s": [
+    0.664,
+    1.748,
+    1.012,
+    0.321
+   ],
+   "first_audio": 6.908,
+   "reply_end": 14.696,
+   "latency_s": 1.276
+  },
+  {
+   "turn": 1,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<ibo>Ee, ọ dị mma.</ibo> <eng>I will leave at six.</eng>",
+   "user_start": 15.396,
+   "user_onset": 15.453,
+   "backchannels": [],
+   "asr": "ee odemma iwelivet six",
+   "asr_s": 0.056,
+   "eot_wait_s": 0.576,
+   "speech_end": 18.048,
+   "target": {
+    "mix": {
+     "eng": 0.95,
+     "ibo": 0.05
+    },
+    "matrix": "eng",
+    "pattern": []
+   },
+   "target_desc": "Match the user's language mix: English about 95%, Igbo about 5% of the words. The main (matrix) language is English.",
+   "reply": "<eng>Yes, six o’clock is fine.</eng> <ibo>Ọ dị mma</ibo> <eng>Shall I confirm the booking now?</eng>",
+   "reply_mix": {
+    "eng": 0.79,
+    "ibo": 0.21
+   },
+   "tvd": 0.162,
+   "how": "first_round",
+   "first_cand_ok": false,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.248,
+     "cands": [
+      {
+       "text": "<eng>Yes, I’ll get a taxi for six. Is it okay if I confirm the pickup time with you?</eng> <ibo>Ee</ibo>",
+       "ok": false,
+       "why": [
+        "non-English tags over English words"
+       ],
+       "tvd": 0.024
+      },
+      {
+       "text": "<eng>Yes, six o’clock is fine. I’ll book the taxi for six.</eng> <ibo>Odemma</ibo>",
+       "ok": false,
+       "why": [
+        "non-English tags over English words"
+       ],
+       "tvd": 0.008
+      },
+      {
+       "text": "<eng>Yes, six o’clock is fine.</eng> <ibo>Ọ dị mma</ibo> <eng>Shall I confirm the booking now?</eng>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.162
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.0033,
+   "llm_s": 0.248,
+   "tts_first_s": 1.168,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.488,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 1.265,
+    "flow_voc": 0.206,
+    "n_tok": 78,
+    "audio_s": 3.12
+   },
+   "tts_chunks_s": [
+    1.168,
+    1.47
+   ],
+   "first_audio": 19.792,
+   "reply_end": 25.312,
+   "latency_s": 1.744
+  },
+  {
+   "turn": 2,
+   "user_label": "TTS-synthesized user turn",
+   "user_ref": "<ibo>Daalụ, ka ọ d���.</ibo>",
+   "user_start": 26.012,
+   "user_onset": 26.077,
+   "backchannels": [],
+   "asr": "dalo kode",
+   "asr_s": 0.043,
+   "eot_wait_s": 0.576,
+   "speech_end": 27.488,
+   "target": {
+    "mix": {
+     "ibo": 0.5,
+     "eng": 0.47
+    },
+    "matrix": "ibo",
+    "pattern": [
+     "closing:ibo",
+     "inter-sentential"
+    ]
+   },
+   "target_desc": "Match the user's language mix: Igbo about 50%, English about 45% of the words. The main (matrix) language is Igbo. The user closed in Igbo: you may close in Igbo.",
+   "reply": "<ibo>Daalụ, ọ dị mma.</ibo> <eng>Thank you, I’ve confirmed your taxi for six.</eng> <ibo>Ka ọ dị.</ibo>",
+   "reply_mix": {
+    "eng": 0.53,
+    "ibo": 0.47
+   },
+   "tvd": 0.047,
+   "how": "first_round",
+   "first_cand_ok": true,
+   "rounds": [
+    {
+     "round": 0,
+     "llm_s": 0.277,
+     "cands": [
+      {
+       "text": "<ibo>Daalụ, biko.</ibo> <eng>Thank you, I’ll confirm your taxi for six o’clock.</eng> <ibo>Ka ọ dị.</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.156
+      },
+      {
+       "text": "<ibo>Daalụ, biko.</ibo> <eng>Thank you, I’ll confirm your taxi for six.</eng> <ibo>Ka ọ dị.</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.133
+      },
+      {
+       "text": "<ibo>Daalụ, ọ dị mma.</ibo> <eng>Thank you, I’ve confirmed your taxi for six.</eng> <ibo>Ka ọ dị.</ibo>",
+       "ok": true,
+       "why": [],
+       "tvd": 0.047
+      }
+     ]
+    }
+   ],
+   "mix_s": 0.002,
+   "llm_s": 0.277,
+   "tts_first_s": 1.131,
+   "spec_start_s": 0.256,
+   "compute_to_first_audio_s": 1.468,
+   "tts_breakdown": {
+    "fe": 0.0,
+    "lm": 0.511,
+    "flow_voc": 0.22,
+    "n_tok": 33,
+    "audio_s": 1.32
+   },
+   "tts_chunks_s": [
+    1.131,
+    1.498,
+    0.731
+   ],
+   "first_audio": 29.212,
+   "reply_end": 35.972,
+   "latency_s": 1.724
+  }
+ ]
+}

samples/05_igbo_heavy.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18334bf0803cd8a34e9c4ed001a026e09e27e44badb2b01adeab91a8a98172fd
+size 586412