Support user-provided speaker audio with a bundled reference encoder

CITATION.cff

@@ -5,6 +5,6 @@ title: "DuDE: Full-Duplex Conversation TTS"
 authors:
   - family-names: Chang
     given-names: Cheng-Kuang
-version: 1.0.0
+version: 1.1.0
 date-released: 2026-09-27
 url: "https://huggingface.co/penguinfish1688/duplexdataengine"

README.md

@@ -199,7 +199,20 @@ and duration are not supported.
 
 ## Voice templates
 
-Four example voices are included for convenience.
+Use your own reference audio for either speaker by passing a file path to
+`--voice-a` and `--voice-b`, or to `voice_a` and `voice_b` in Python:
+
+```bash
+python infer.py --model . --xml dialogue.xml \
+  --voice-a speaker_a.wav --voice-b speaker_b.wav --output dialogue.wav
+```
+
+Use a clear, single-speaker WAV or FLAC clip between 0.5 and 60 seconds long.
+No transcript is needed. Audio is converted to mono and resampled automatically.
+You can mix a custom reference with an included voice, such as
+`--voice-a speaker_a.wav --voice-b voice_2`.
+
+If you do not have reference audio, four example voices are included for convenience:
 
 | Template | Reference audio |
 | --- | --- |

dude_tts/runtime.py

@@ -1,4 +1,4 @@
-"""Portable XML + two voice presets -> normalized 24 kHz stereo audio."""
+"""XML + two reference voices -> normalized 24 kHz stereo audio."""
 from contextlib import nullcontext
 from dataclasses import dataclass
 import json
@@ -18,6 +18,7 @@ from .audio import normalize_for_listening
 from .frontend import duplex_prefix
 from .interleave import EVENTS
 from .model import DualChannelTTS
+from .voice import ReferenceVoiceEncoder
 
 
 @dataclass
@@ -43,7 +44,8 @@ class DuDE:
         root = Path(model_id)
         if not root.is_dir():
             root = Path(snapshot_download(model_id, revision=revision,
-                allow_patterns=['*.json', '*.txt', '*.safetensors', 'speech_tokenizer/*', 'voices/*']))
+                allow_patterns=['*.json', '*.txt', '*.safetensors', 'speech_tokenizer/*',
+                                'speaker_encoder/*', 'voices/*']))
         return cls(root, device=device)
 
     def __init__(self, root, *, device='cuda'):
@@ -57,6 +59,8 @@ class DuDE:
         self.release = json.loads((self.root/'release.json').read_text())
         self.events = json.loads((self.root/'event-token-map.json').read_text())
         self.voices = json.loads((self.root/'voices/presets.json').read_text())
+        self._voice_encoder = None
+        self._voice_cache = {}
         config = Qwen3TTSConfig.from_pretrained(self.root, local_files_only=True)
         config.talker_config._attn_implementation = 'sdpa'
         config.talker_config.code_predictor_config._attn_implementation = 'sdpa'
@@ -85,6 +89,24 @@ class DuDE:
     def _event(self, event):
         return [self.events[event]]
 
+    def voice_embedding(self, reference):
+        """Accept an included voice name or a path to a single-speaker recording."""
+        if isinstance(reference, str) and reference in self.voices:
+            return self.voices[reference]['embedding']
+        if not isinstance(reference, (str, Path)):
+            raise ValueError('A voice must be an audio file path or an included voice name.')
+        path = Path(reference).expanduser().resolve()
+        if not path.is_file():
+            raise ValueError(f'Reference audio {str(reference)!r} does not exist. '
+                             f'Provide an audio file or choose from {list(self.voices)}.')
+        stat = path.stat()
+        key = (str(path), stat.st_mtime_ns, stat.st_size)
+        if key not in self._voice_cache:
+            if self._voice_encoder is None:
+                self._voice_encoder = ReferenceVoiceEncoder(self.root/'speaker_encoder', self.device)
+            self._voice_cache[key] = self._voice_encoder.encode(path)
+        return self._voice_cache[key]
+
     def conditioning(self, xml, voice_a, voice_b):
         if not isinstance(xml, str) or not xml.strip() or len(xml) > 30000:
             raise ValueError('Enter a nonempty dialogue XML, up to 30,000 characters.')
@@ -92,9 +114,7 @@ class DuDE:
                       instruct_ids=[], language='English', speaker='Ryan')
         channels = {}
         for channel, voice in zip('AB', [voice_a, voice_b]):
-            if voice not in self.voices:
-                raise ValueError(f'Unknown voice {voice!r}; choose from {list(self.voices)}')
-            channels[channel] = dict(header, voice_embedding=self.voices[voice]['embedding'])
+            channels[channel] = dict(header, voice_embedding=self.voice_embedding(voice))
         record = dict(xml=xml, channels=channels)
         conditioning = duplex_prefix(record, self.model.config, self._text, self._event)
         if len(conditioning['text_ids']) > 12000:

dude_tts/voice.py

@@ -0,0 +1,57 @@
+"""Extract reference voices with the same frozen encoder used during training."""
+import json
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+import torch
+from safetensors.torch import load_file
+from scipy.signal import resample_poly
+
+
+def reference_audio(path):
+    """Read a single-speaker clip, mix to mono, and resample to 24 kHz."""
+    path = Path(path).expanduser()
+    try:
+        info = sf.info(path)
+        if not .5 <= info.duration <= 60:
+            raise ValueError('Use a reference clip between 0.5 and 60 seconds long.')
+        wave, rate = sf.read(path, dtype='float32', always_2d=True)
+    except (sf.LibsndfileError, OSError) as error:
+        raise ValueError(f'Cannot read reference audio {path.name!r}; use a WAV or FLAC file.') from error
+    if not np.isfinite(wave).all():
+        raise ValueError('Reference audio contains nonfinite samples.')
+    wave = wave.mean(axis=1)
+    if not np.any(wave):
+        raise ValueError('Reference audio is silent; choose a clip containing speech.')
+    if rate != 24000:
+        wave = resample_poly(wave, 24000, rate)
+    return np.asarray(wave, dtype=np.float32)
+
+
+class ReferenceVoiceEncoder:
+    def __init__(self, directory, device):
+        from qwen_tts.core.models.configuration_qwen3_tts import Qwen3TTSSpeakerEncoderConfig
+        from qwen_tts.core.models.modeling_qwen3_tts import Qwen3TTSSpeakerEncoder
+        directory = Path(directory)
+        if not (directory/'model.safetensors').is_file():
+            raise ValueError('The speaker encoder is missing. Download the current complete model bundle.')
+        config = Qwen3TTSSpeakerEncoderConfig(**json.loads((directory/'config.json').read_text()))
+        self.device = torch.device(device)
+        self.dtype = torch.bfloat16 if self.device.type == 'cuda' else torch.float32
+        self.model = Qwen3TTSSpeakerEncoder(config)
+        self.model.load_state_dict(load_file(str(directory/'model.safetensors')), strict=True)
+        self.model.to(device=self.device, dtype=self.dtype).eval()
+        self.dimension = config.enc_dim
+
+    @torch.inference_mode()
+    def encode(self, path):
+        from qwen_tts.core.models.modeling_qwen3_tts import mel_spectrogram
+        wave = reference_audio(path)
+        mel = mel_spectrogram(torch.from_numpy(wave)[None], n_fft=1024,
+            num_mels=128, sampling_rate=24000, hop_size=256, win_size=1024,
+            fmin=0, fmax=12000).transpose(1, 2)
+        embedding = self.model(mel.to(self.device, dtype=self.dtype))[0].float().cpu()
+        if embedding.shape != (self.dimension,) or not torch.isfinite(embedding).all():
+            raise ValueError('Could not extract a valid speaker embedding from the reference clip.')
+        return embedding.tolist()

infer.py

@@ -8,8 +8,8 @@ if __name__ == '__main__':
     p = argparse.ArgumentParser(description=__doc__)
     p.add_argument('--model', default='penguinfish1688/duplexdataengine')
     p.add_argument('--xml', required=True, help='Path to a dialogue XML file')
-    p.add_argument('--voice-a', default='voice_1')
-    p.add_argument('--voice-b', default='voice_2')
+    p.add_argument('--voice-a', default='voice_1', help='Speaker A reference audio path or included voice name')
+    p.add_argument('--voice-b', default='voice_2', help='Speaker B reference audio path or included voice name')
     p.add_argument('--output', default='duplex.wav')
     p.add_argument('--seed', type=int, default=20260922)
     p.add_argument('--max-seconds', type=float, default=150.)

release.json

@@ -1,6 +1,6 @@
 {
   "format": "dude_duplex_v1",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "author": "Cheng-Kuang Chang",
   "base_model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
   "base_revision": "0c0e3051f131929182e2c023b9537f8b1c68adfe",
@@ -46,5 +46,11 @@
       "source_conversation": "V01_S0220_I00000126"
     }
   ],
-  "license": "cc-by-nc-4.0"
+  "license": "cc-by-nc-4.0",
+  "speaker_encoder": {
+    "source_model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+    "revision": "fd4b254389122332181a7c3db7f27e918eec64e3",
+    "weights": "speaker_encoder/model.safetensors",
+    "license": "apache-2.0"
+  }
 }

requirements.txt

@@ -6,4 +6,5 @@ safetensors>=0.5
 huggingface_hub>=0.34,<1
 numpy>=1.26
 soundfile>=0.12
+scipy>=1.11
 qwen-tts @ git+https://github.com/QwenLM/Qwen3-TTS.git@022e286b98fbec7e1e916cb940cdf532cd9f488e

speaker_encoder/config.json

@@ -0,0 +1,4 @@
+{
+  "enc_dim": 2048,
+  "sample_rate": 24000
+}

speaker_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df60a638e7f4a29331c0af2bd2984ee5b992fee9d5923c776f7e4bdc3dedea48
+size 24010000