File size: 50,218 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
"""Native incremental speech primitives retained for the formal HFD runtime.

Native Qwen2.5-Omni weights are reused. There is no external TTS and no
whole-utterance waveform to slice. A stateful code->mel->PCM decoder accepts
new codec tokens, emits committed audio, and retains overlap CONTEXT only.

The window geometry is adapted from QwenLM/Qwen2.5-Omni (Apache-2.0), commit
d8a31ca56c0456b6edfcbcbf4bdbb6ae2200ef42, low-VRAM-mode. Copyright 2025
Qwen Team / HuggingFace Inc. Unlike that example we expose PCM as it is made,
never delete model modules, and retain the installed 4.57 RK4 solver by default.
The formal default is selected by :class:`psi.runtime.speech.SpeechRuntimeConfig`:
Euler-6, 36 codec codes and a 0.20-second playback threshold.  The module is
optional; importing ``psi.runtime`` does not import model-specific helpers.
"""
from dataclasses import dataclass, replace
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager, nullcontext
import threading
import time

import torch
from transformers.cache_utils import DynamicCache

REPLY_QUERY = (
    '<|im_end|>\n<|im_start|>user\nAnswer the latest spoken visual question '
    'with a short phrase. Use the camera view when it was asked.<|im_end|>\n'
    '<|im_start|>assistant\n'
)


def _clone_selected_cache(past, rows):
    return DynamicCache(ddp_cache_data=[
        (layer.keys.index_select(0, rows), layer.values.index_select(0, rows))
        for layer in past.layers
    ])


@dataclass(frozen=True)
class SpeechGeometry:
    chunk_codes: int = 48
    left_codes: int = 24
    right_codes: int = 12
    vocoder_left: int = 20
    vocoder_right: int = 20
    sample_rate: int = 24000
    hop: int = 240
    max_mel_frames: int = 30000


class PCMClockPackets:
    """Reblock already-produced audio onto the action clock, never a full WAV.

    At 24 kHz, 9,600 NEW samples correspond to one 0.4 s action step. The
    speech decoder may have a different native block size; this small FIFO
    reconciles the clocks without resetting its hidden/context states.
    """
    def __init__(self, samples_per_packet=9600):
        assert samples_per_packet > 0
        self.size = samples_per_packet
        self.buffer = torch.empty(0, dtype=torch.float32)
        self.received = 0
        self.emitted = 0
        self.closed = False
        self.last_event = None

    def push(self, pcm, event):
        if self.closed:
            raise RuntimeError('PCM clock already closed')
        assert pcm.device.type == 'cpu' and pcm.ndim == 1
        assert event['sample_start'] == self.received
        self.received += pcm.numel()
        assert event['sample_end'] == self.received
        self.last_event = dict(event)
        self.buffer = torch.cat((self.buffer, pcm.float()))
        result = []
        while self.buffer.numel() >= self.size:
            data = self.buffer[:self.size].clone()
            self.buffer = self.buffer[self.size:]
            row = dict(event, sample_start=self.emitted, samples=data.numel(),
                       sample_end=self.emitted + data.numel(),
                       available_monotonic=time.monotonic(), partial_final=False)
            self.emitted += data.numel()
            result.append((data, row))
        return result

    def finish(self):
        if self.closed:
            raise RuntimeError('PCM clock already closed')
        self.closed = True
        result = []
        if self.buffer.numel():
            row = dict(self.last_event, sample_start=self.emitted,
                       samples=self.buffer.numel(), sample_end=self.received,
                       available_monotonic=time.monotonic(), partial_final=True)
            result.append((self.buffer.clone(), row))
            self.emitted = self.received
        assert self.emitted == self.received
        self.buffer = self.buffer[:0]
        return result


class IncrementalCode2Wav:
    """One utterance, B=1. push/finish return NEW, nonoverlapping PCM chunks.

    Code context is not reset at each chunk; noise uses global mel coordinates.
    No cross-fade is applied to conceal decoder errors. Boundary quality must
    be measured and listened to. This class does not itself enforce wall pacing.
    """
    def __init__(self, token2wav, conditioning, reference_mel, *, seed,
                 geometry=None, solver='rk4', num_steps=10, guidance_scale=.5,
                 sway_coefficient=-1., sample_fn=None, trace=None,
                 first_chunk_codes=None):
        self.geometry = geometry or SpeechGeometry()
        self.regular_geometry = self.geometry
        first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None
                             else int(first_chunk_codes))
        self.first_geometry = replace(self.geometry, chunk_codes=first_chunk_codes)
        self.dit = token2wav.code2wav_dit_model
        self.vocoder = token2wav.code2wav_bigvgan_model
        self.conditioning = conditioning.float()
        self.reference = reference_mel.float()
        self.device = self.reference.device
        self.solver = solver
        self.num_steps = num_steps
        self.guidance_scale = guidance_scale
        self.sway = sway_coefficient
        self.sample_fn = sample_fn
        self.trace = trace
        self.repeats = int(self.dit.repeats)
        g = self.geometry
        assert self.reference.shape[0] == self.conditioning.shape[0] == 1
        assert g.chunk_codes > 0 and g.left_codes >= 0 and g.right_codes >= 0
        # Global block alignment is essential for the native block-attention mask.
        assert (g.chunk_codes * self.repeats) % self.dit.block_size == 0
        assert (g.left_codes * self.repeats) % self.dit.block_size == 0
        assert (self.first_geometry.chunk_codes * self.repeats) % self.dit.block_size == 0
        assert solver in ('rk4', 'euler') and num_steps >= 2
        assert g.vocoder_left >= 0 and g.vocoder_right >= 0
        generator = torch.Generator(device='cpu').manual_seed(seed)
        # Generate once on CPU for deterministic compatibility with the old
        # implementation, then move the complete noise bank to the decoder
        # device. The previous code copied one slice CPU->GPU on every
        # Token2Wav block, which put a host transfer on the critical path.
        noise = torch.randn(1, g.max_mel_frames, self.dit.mel_dim,
                            generator=generator, dtype=self.reference.dtype)
        self.noise = noise.to(self.device)
        del noise
        self.code = torch.empty(1, 0, device=self.device, dtype=torch.long)
        self.mel = torch.empty(1, self.dit.mel_dim, 0, device=self.device)
        self.code_left = 0
        self.mel_left = 0
        self.code_start = 0
        self.received = 0
        self.committed_codes = 0
        self.committed_mel = 0
        self.emitted_samples = 0
        self.finished = False
        self.events = []

    def _sample(self, code, start):
        if self.trace is not None:
            self.trace.mark_once('token2wav_first_start')
        count = code.shape[1] * self.repeats
        if start < 0 or start + count > self.noise.shape[1]:
            raise ValueError('Utterance exceeds explicit noise/context capacity')
        initial = self.noise[:, start:start + count]
        if self.sample_fn is not None:
            result = self.sample_fn(code, initial, start)
            if self.trace is not None:
                self.trace.mark_once('token2wav_first_end')
            return result
        condition = self.conditioning[:, None].expand(-1, count, -1)

        def velocity(t, state):
            output = self.dit(hidden_states=state, quantized_code=code,
                speaker_embedding=condition, condition_vector=self.reference,
                time_step=t, apply_cfg=self.guidance_scale >= 1e-5)
            if self.guidance_scale < 1e-5:
                return output
            guided, null = output.chunk(2, dim=0)
            return guided + (guided - null) * self.guidance_scale

        times = torch.linspace(0, 1, self.num_steps, device=self.device,
                               dtype=self.reference.dtype)
        if self.sway is not None:
            times += self.sway * (torch.cos(torch.pi / 2 * times) - 1 + times)
        if self.solver == 'rk4':
            from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import RungeKutta4ODESolver
            result = RungeKutta4ODESolver(velocity, initial).integrate(times)[-1]
        else:
            result = initial
            for t0, t1 in zip(times[:-1], times[1:]):
                result = result + velocity(t0, result) * (t1 - t0)
        result = result.transpose(1, 2)
        if self.trace is not None:
            self.trace.mark_once('token2wav_first_end')
        return result

    def _vocode(self, new_mel, *, final=False):
        g = self.geometry
        self.mel = torch.cat((self.mel, new_mel), dim=-1)
        right = 0 if final else g.vocoder_right
        frames = self.mel.shape[-1]
        if frames <= self.mel_left + right:
            return []
        wave = self.vocoder(self.mel).reshape(-1)
        assert wave.numel() == frames * g.hop, 'Unexpected native vocoder hop'
        end = wave.numel() - right * g.hop
        pcm = wave[self.mel_left * g.hop:end].float().detach().cpu().clone()
        assert pcm.numel() and torch.isfinite(pcm).all()
        if self.trace is not None:
            # This is the first point at which Token2Wav has produced actual
            # PCM samples.  The caller may still need to write/enqueue them.
            self.trace.mark_once('talker_first_chunk_ready')
        # .cpu() is intentionally a completion boundary: these samples are ready
        # to play now, not merely queued CUDA operations with premature timestamps.
        event = dict(index=len(self.events), codec_received=self.received,
            code_prefix_committed=self.committed_codes,
            sample_start=self.emitted_samples, samples=pcm.numel(),
            sample_end=self.emitted_samples + pcm.numel(), final=final,
            available_monotonic=time.monotonic())
        self.events.append(event)
        self.emitted_samples += pcm.numel()
        if not final:
            keep = min(frames, g.vocoder_left + g.vocoder_right)
            self.mel = self.mel[:, :, -keep:] if keep else self.mel[:, :, :0]
            self.mel_left = keep - g.vocoder_right
            assert self.mel_left >= 0
        else:
            self.mel = self.mel[:, :, :0]
        return [(pcm, event)]

    @torch.inference_mode()
    def push(self, tokens):
        if self.finished:
            raise RuntimeError('Cannot append to a finished speech stream')
        values = torch.as_tensor(tokens, dtype=torch.long, device=self.device).reshape(-1)
        output = []
        for token in values:
            g = (self.first_geometry if self.committed_codes == 0
                 else self.regular_geometry)
            self.geometry = g
            self.code = torch.cat((self.code, token.reshape(1, 1)), dim=1)
            self.received += 1
            if self.received * self.repeats > g.max_mel_frames:
                raise ValueError('Utterance exceeds explicit noise/context capacity')
            target = self.code_left + g.chunk_codes + g.right_codes
            if self.code.shape[1] < target:
                continue
            assert self.code.shape[1] == target
            mel = self._sample(self.code, self.code_start * self.repeats)
            first = self.code_left * self.repeats
            last = (self.code.shape[1] - g.right_codes) * self.repeats
            new = mel[:, :, first:last]
            assert new.shape[-1] == g.chunk_codes * self.repeats
            self.committed_codes += g.chunk_codes
            self.committed_mel += new.shape[-1]
            output.extend(self._vocode(new))
            keep = min(self.code.shape[1], g.left_codes + g.right_codes)
            self.code = self.code[:, -keep:] if keep else self.code[:, :0]
            self.code_left = keep - g.right_codes
            self.code_start = self.committed_codes - self.code_left
        self.geometry = self.regular_geometry
        return output

    @torch.inference_mode()
    def finish(self):
        if self.finished:
            raise RuntimeError('Speech stream already finished')
        self.finished = True
        remaining = self.received - self.committed_codes
        output = []
        if remaining:
            mel = self._sample(self.code, self.code_start * self.repeats)
            new = mel[:, :, self.code_left * self.repeats:]
            assert new.shape[-1] == remaining * self.repeats
            self.committed_codes += remaining
            self.committed_mel += new.shape[-1]
            # Flush all vocoder right context exactly once at utterance end.
            output.extend(self._vocode(new, final=True))
        elif self.mel.shape[-1] > self.mel_left:
            empty = self.mel[:, :, :0]
            output.extend(self._vocode(empty, final=True))
        assert self.committed_codes == self.received
        assert self.emitted_samples == self.received * self.repeats * self.geometry.hop
        self.code = self.code[:, :0]
        return output


class BufferedPCMDecoder:
    """Bounded, ordered codec->PCM worker; no future tokens or full WAV.

    The producer submits only newly available native chunks. One decoder owns
    all context/noise state in one worker thread. At most two jobs may be
    outstanding; backpressure never drops codes. Callback completion is joined
    before close returns, including on producer/decoder failure.
    """
    def __init__(self, factory, on_pcm, *, geometry=None,
                 context_factory=nullcontext, max_pending=2, trace=None,
                 first_chunk_codes=None):
        self.geometry = geometry or SpeechGeometry()
        if max_pending < 1:
            raise ValueError('At least one outstanding decoder job is required')
        self.factory, self.on_pcm = factory, on_pcm
        self.context_factory = context_factory
        self.max_pending = max_pending
        self.trace = trace
        self.pending = deque()
        self.buffer = []
        self.first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None
                                  else int(first_chunk_codes))
        self.target = self.first_chunk_codes + self.geometry.right_codes
        self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='native-pcm')
        self.decoder = None
        self.failure = None
        self.failure_observed = False
        self.finished = False
        self.closed = False
        self.producer_done = threading.Event()
        self.codec_eos = False
        self.max_pending_observed = 0
        self.backpressure_seconds = 0.

    def mark_codec_finished(self, eos):
        self.codec_eos = bool(eos)
        self.producer_done.set()

    def _run(self, values, final):
        if self.failure is not None:
            raise RuntimeError('Decoder pipeline failed on an earlier chunk') from self.failure
        try:
            with self.context_factory():
                if self.decoder is None:
                    self.decoder = self.factory()
                output = self.decoder.push(values)
                if final:
                    output.extend(self.decoder.finish())
                for pcm, event in output:
                    # In a pipeline the producer may reach EOS while this worker
                    # decodes an earlier chunk. Report actual completion, not the
                    # state when the chunk was enqueued.
                    event['codec_eos_observed'] = self.producer_done.is_set() and self.codec_eos
                    event['codec_generation_finished'] = self.producer_done.is_set()
                    self.on_pcm(pcm, event)
        except BaseException as error:
            self.failure = error
            raise

    def _result(self, future):
        try:
            return future.result()
        except BaseException:
            self.failure_observed = True
            raise

    def _submit(self, values, final=False):
        while self.pending and self.pending[0].done():
            self._result(self.pending.popleft())
        if len(self.pending) >= self.max_pending:
            start = time.monotonic()
            self._result(self.pending.popleft())
            self.backpressure_seconds += time.monotonic() - start
        self.pending.append(self.executor.submit(self._run, tuple(values), final))
        self.max_pending_observed = max(self.max_pending_observed, len(self.pending))
        if self.trace is not None:
            self.trace.set('speech_queue_depth', len(self.pending))
            self.trace.set('speech_queue_max_depth', self.max_pending_observed)

    def push(self, value):
        if self.closed or self.finished:
            raise RuntimeError('Cannot append to a closed decoder pipeline')
        self.buffer.append(int(value))
        if len(self.buffer) == self.target:
            self._submit(self.buffer)
            self.buffer = []
            self.target = self.geometry.chunk_codes

    def finish(self):
        if self.closed or self.finished:
            raise RuntimeError('Decoder pipeline already finished')
        self.finished = True
        self._submit(self.buffer, final=True)
        self.buffer = []
        while self.pending:
            self._result(self.pending.popleft())

    @property
    def events(self):
        return [] if self.decoder is None else self.decoder.events

    @property
    def emitted_samples(self):
        return 0 if self.decoder is None else self.decoder.emitted_samples

    def close(self):
        if not self.closed:
            self.closed = True
            # On failure cancel only jobs not started yet; join the running
            # callback before its caller closes the audio sink or model.
            self.executor.shutdown(wait=True, cancel_futures=True)
        if self.failure is not None and not self.failure_observed:
            self.failure_observed = True
            # If the producer also failed, Python retains it as __context__ in
            # the traceback. Do not silently hide a concurrent audio failure.
            raise self.failure


class IncrementalAnswer:
    """Greedy Thinker side branch; one causal text feature returned per pop().

    No GT text and no future camera/audio enter this branch. The input history
    is cloned, so ongoing perception/action may continue appending its own KV.
    """
    @torch.inference_mode()
    def __init__(self, model, cache, max_tokens=48):
        self.model = model
        self.thinker = model.vlm_model
        self.device = next(model.reply_head.parameters()).device
        self.max_tokens = max_tokens
        self.generated = []
        self.tape = []
        self.done = False
        self.eos = False
        self.complete_monotonic = None
        self.source_cache = cache
        self.source_length = cache.token_count
        self.past = _clone_selected_cache(cache.past_key_values,
                                        torch.tensor([0], device=self.device))
        ids = torch.tensor([model.vlm_processor.tokenizer.encode(
            REPLY_QUERY, add_special_tokens=False)], device=self.device)
        n = ids.shape[1]
        self.offset = cache.av_next_positions[0] + n
        if cache.token_count + n + max_tokens > model.av_context_limit:
            raise ValueError('Incremental answer exceeds context capacity')
        self.mask = torch.cat((cache.attention_mask[:1], torch.ones_like(ids)), 1)
        emb = self.thinker.get_input_embeddings()(ids)
        self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask,
            position_ids=(torch.arange(n, device=self.device) + cache.av_next_positions[0])[None,None].expand(3,1,-1),
            past_key_values=self.past,
            cache_position=torch.arange(cache.token_count, cache.token_count+n, device=self.device),
            use_cache=True, return_dict=True)
        self.past = self.out.past_key_values
        self.query = (ids, emb, self.out.last_hidden_state)

    @torch.inference_mode()
    def pop(self):
        if self.done:
            return None
        token = self.thinker.lm_head(self.out.last_hidden_state[:, -1:]).argmax(-1)
        value = int(token.item())
        self.generated.append(value)
        self.eos = value == int(self.thinker.config.eos_token_id)
        if self.eos:
            self.done = True
            self.complete_monotonic = time.monotonic()
            return None
        emb = self.thinker.get_input_embeddings()(token)
        count = self.past.get_seq_length()
        self.mask = torch.cat((self.mask, torch.ones_like(token)), 1)
        self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask,
            past_key_values=self.past, position_ids=torch.full((3,1,1), self.offset,
                device=self.device, dtype=torch.long),
            cache_position=torch.tensor([count], device=self.device), use_cache=True, return_dict=True)
        self.past = self.out.past_key_values
        self.offset += 1
        piece = (token, emb, self.out.last_hidden_state)
        self.tape.append(piece)
        if len(self.generated) >= self.max_tokens:
            self.done = True
            self.complete_monotonic = time.monotonic()
        return piece

    def result(self):
        assert self.done
        assert self.source_cache.past_key_values.get_seq_length() == self.source_length
        return dict(text=self.model.vlm_processor.tokenizer.decode(self.generated, skip_special_tokens=True),
            ids=self.generated, eos=self.eos, branch_tape=[self.query] + self.tape,
            text_complete_monotonic=self.complete_monotonic)


class NativeTextConditionAnswer:
    """The trained AV model chooses words; frozen native features drive speech.

    Only tokens already emitted by the trained model enter the frozen voice KV.
    No GT answer, future input, or trained hidden-state distribution is passed
    to the frozen Talker. Both private caches leave the action cache untouched.
    """
    @torch.inference_mode()
    def __init__(self, model, cache, voice_thinker, max_tokens=48):
        self.learner = IncrementalAnswer(model, cache, max_tokens)
        self.voice = voice_thinker
        self.device = self.learner.device
        tokenizer = model.vlm_processor.tokenizer
        ids = torch.tensor([tokenizer.apply_chat_template([
            dict(role='system', content='You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'),
            dict(role='user', content='Give a short spoken answer.')],
            tokenize=True, add_generation_prompt=True)], device=self.device)
        self.mask = torch.ones_like(ids)
        emb = self.voice.get_input_embeddings()(ids)
        self.offset = ids.shape[1]
        out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask,
            position_ids=torch.arange(self.offset, device=self.device)[None, None].expand(3, 1, -1),
            cache_position=torch.arange(self.offset, device=self.device), use_cache=True, return_dict=True)
        self.past = out.past_key_values
        self.query = (ids, emb, out.last_hidden_state)
        self.arrived_ids = []

    @property
    def done(self):
        return self.learner.done

    @torch.inference_mode()
    def pop(self):
        piece = self.learner.pop()
        if piece is None:
            return None
        ids = piece[0]
        self.arrived_ids.extend(ids[0].tolist())
        emb = self.voice.get_input_embeddings()(ids)
        self.mask = torch.cat((self.mask, torch.ones_like(ids)), 1)
        count = self.past.get_seq_length()
        out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask,
            position_ids=torch.full((3, 1, 1), self.offset, device=self.device, dtype=torch.long),
            cache_position=torch.tensor([count], device=self.device),
            past_key_values=self.past, use_cache=True, return_dict=True)
        self.past = out.past_key_values
        self.offset += 1
        return ids, emb, out.last_hidden_state

    def result(self):
        result = self.learner.result()
        expected = [t for t in result['ids'] if t != int(self.learner.thinker.config.eos_token_id)]
        if self.arrived_ids != expected:
            raise RuntimeError('Native voice did not consume exactly the generated answer tokens')
        return dict(result, voice_conditioning='frozen_base_text_causal', voice_arrived_ids=self.arrived_ids)


def _hidden_map(hidden, delta):
    if delta is None:
        return hidden
    # Match the train-only fitted interface exactly. The caller alternates
    # BF16 Thinker/Talker and FP32 decoder contexts; outer autocast must NOT
    # silently quantize this FP32 matrix multiply on later text tokens.
    shape = hidden.shape
    with torch.autocast(device_type=hidden.device.type, enabled=False):
        x = hidden.float().reshape(-1, shape[-1])
        mapped = x + x @ delta[:-1] + delta[-1]
    return mapped.reshape(shape).to(hidden.dtype)


class FusedFlashTalkerKV:
    """Fixed-capacity Talker KV using FlashAttention's decode kernel.

    The native prefill remains untouched.  This object only replaces the
    one-token DynamicCache growth path, which otherwise reallocates and
    concatenates every layer's full K/V history at every codec token.
    """
    def __init__(self, dynamic_cache, *, max_length):
        from flash_attn import flash_attn_with_kvcache
        self.flash_attn_with_kvcache = flash_attn_with_kvcache
        self.length = int(dynamic_cache.get_seq_length())
        if self.length >= max_length:
            raise ValueError('Fused Talker cache capacity must exceed prefill length')
        self.max_length = int(max_length)
        self.keys = []
        self.values = []
        for layer in dynamic_cache.layers:
            # HF cache: [B, Hkv, S, D]. Flash decode cache: [B, S, Hkv, D].
            key = layer.keys.transpose(1, 2).contiguous()
            value = layer.values.transpose(1, 2).contiguous()
            key_store = torch.empty(
                (key.shape[0], self.max_length, key.shape[2], key.shape[3]),
                dtype=key.dtype, device=key.device)
            value_store = torch.empty_like(key_store)
            key_store[:, :self.length].copy_(key)
            value_store[:, :self.length].copy_(value)
            self.keys.append(key_store)
            self.values.append(value_store)
        self.cache_seqlens = torch.full(
            (self.keys[0].shape[0],), self.length,
            dtype=torch.int32, device=self.keys[0].device)

    def get_seq_length(self):
        return self.length

    def _step_impl(self, talker, hidden_states, position_ids):
        from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
            apply_multimodal_rotary_pos_emb,
        )
        position_embeddings = talker.model.rotary_emb(hidden_states, position_ids)
        for index, layer in enumerate(talker.model.layers):
            residual = hidden_states
            normed = layer.input_layernorm(hidden_states)
            attn = layer.self_attn
            batch, query_length, _ = normed.shape
            query = attn.q_proj(normed).view(
                batch, query_length, attn.num_heads, attn.head_dim)
            key = attn.k_proj(normed).view(
                batch, query_length, attn.num_key_value_heads, attn.head_dim)
            value = attn.v_proj(normed).view(
                batch, query_length, attn.num_key_value_heads, attn.head_dim)
            cos, sin = position_embeddings
            query, key = apply_multimodal_rotary_pos_emb(
                query.transpose(1, 2), key.transpose(1, 2), cos, sin,
                attn.rope_scaling['mrope_section'])
            query = query.transpose(1, 2).contiguous()
            key = key.transpose(1, 2).contiguous()
            attended = self.flash_attn_with_kvcache(
                query, self.keys[index], self.values[index],
                k=key, v=value, cache_seqlens=self.cache_seqlens,
                softmax_scale=attn.scaling, causal=True, num_splits=1)
            attended = attended.reshape(batch, query_length, -1).contiguous()
            hidden_states = residual + attn.o_proj(attended)
            residual = hidden_states
            hidden_states = residual + layer.mlp(
                layer.post_attention_layernorm(hidden_states))
        return talker.model.norm(hidden_states)

    def step(self, talker, hidden_states, position_ids):
        """Run exactly one causal Talker token and append its K/V in-place."""
        if hidden_states.shape[1] != 1:
            raise ValueError('Fused Talker decode only supports one new token')
        if self.length >= self.max_length:
            raise RuntimeError('Fused Talker KV cache is full')
        hidden_states = self._step_impl(talker, hidden_states, position_ids)
        self.length += 1
        self.cache_seqlens.fill_(self.length)
        return hidden_states


@torch.inference_mode()
def stream_native_reply(native, model, cache, prompt_tape, *, seed,
                        on_pcm, speaker='Chelsie', adapter=None,
                        geometry=None, solver='rk4', num_steps=10,
                        max_codec_tokens=500,
                        code_only=False, do_sample=True, parallel_decoder=False,
                        trace=None, decoder_device=None, first_chunk_codes=None,
                        talker_cache='dynamic', compile_talker=False,
                        compile_dynamic=False,
                        codec_sync_interval=1, preallocate_sequence=False,
                        fast_talker_step=False, fast_sampling=False,
                        fused_flash_kv=False, native_text_conditioning=False):
    """Interleave Thinker tokens and native Talker tokens; callback NEW PCM.

    This function belongs in a separate worker/CUDA stream while the body loop
    runs. Its return contains metrics and codec IDs, NOT a hidden full WAV.
    No whole text answer is generated before Talker starts. Tiny answers may
    naturally finish before the decoder's finite-lookahead first audio chunk.
    parallel_decoder overlaps codec production and PCM decoding on independent
    CUDA streams, with a bounded FIFO; default False preserves the serial arm.
    """
    from transformers.generation.logits_process import (
        LogitsProcessorList, RepetitionPenaltyLogitsProcessor,
        SuppressTokensLogitsProcessor, TemperatureLogitsWarper,
        TopKLogitsWarper, TopPLogitsWarper)
    before = time.monotonic()
    if codec_sync_interval < 1:
        raise ValueError('codec_sync_interval must be positive')
    if trace is not None:
        trace.mark_once('speech_job_start')
    learner_device = model.device
    talker = native.talker
    device = next(talker.parameters()).device
    decoder_device = torch.device(device if decoder_device is None else decoder_device)
    talker.rope_deltas = None  # reset per utterance, not per chunk
    dtype = talker.thinker_to_talker_proj.weight.dtype
    delta = adapter.to(device=device, dtype=torch.float32) if adapter is not None else None
    if trace is not None:
        trace.mark_once('native_thinker_start')
    with torch.autocast('cuda', dtype=torch.bfloat16):
        answer = (NativeTextConditionAnswer(model, cache, native.voice_thinker)
                  if native_text_conditioning else IncrementalAnswer(model, cache))
        first = answer.pop()
    if first is not None and trace is not None:
        trace.mark_once('native_thinker_first_hidden')
    if first is None:
        return dict(native_speech=False, native_speech_error='empty generated answer',
                    **answer.result())
    if native_text_conditioning:
        delta = None
    tape = [answer.query] if native_text_conditioning else list(prompt_tape) + [answer.query]
    ids = torch.cat([x[0] for x in tape], 1)
    emb = torch.cat([x[1] for x in tape], 1).to(dtype).clone()
    # Use exactly the fitted/evaluated per-packet GEMM shapes. Mapping one
    # concatenated history instead changes floating-point kernel/reduction
    # choices; BF16 rounding then perturbs autoregressive sampling later on.
    hidden = torch.cat([_hidden_map(x[2], delta).to(dtype) for x in tape], 1)
    # Same masking as native.generate: AV semantics are carried by hidden,
    # not reintroduced as raw frontend embeddings into Talker.
    for name in ('audio_token_id', 'image_token_id', 'video_token_id'):
        emb.masked_fill_(ids.eq(getattr(model.vlm_model.config, name))[:, :, None], 0)
    speaker_params = native.speaker_map[speaker]
    # In the split-GPU path, the frozen voice Thinker and Talker live together
    # while the AV learner keeps its source KV on the body GPU.  The embedding
    # weights are identical, but using the voice copy avoids a device roundtrip.
    embedder = (native.voice_thinker.get_input_embeddings()
                if hasattr(native, 'voice_thinker') else
                model.vlm_model.get_input_embeddings())
    text_bos = torch.tensor([[speaker_params['bos_token']]], device=device)
    input_text_ids = torch.cat((ids, text_bos, first[0]), 1)
    input_ids = torch.cat((torch.full_like(ids, talker.codec_mask_token),
        torch.tensor([[talker.codec_pad_token, talker.codec_bos_token]], device=device)), 1)
    first_feature = _hidden_map(first[2], delta).to(dtype) + first[1].to(dtype)
    inputs_embeds = torch.cat((hidden + emb, embedder(text_bos).to(dtype), first_feature), 1)
    mask = torch.ones_like(input_ids)
    if preallocate_sequence:
        initial_sequence_length = input_ids.shape[1]
        input_ids_storage = torch.empty(
            (input_ids.shape[0], initial_sequence_length + max_codec_tokens),
            dtype=input_ids.dtype, device=input_ids.device)
        mask_storage = torch.zeros_like(input_ids_storage)
        input_ids_storage[:, :initial_sequence_length].copy_(input_ids)
        mask_storage[:, :initial_sequence_length].copy_(mask)
        active_sequence_length = initial_sequence_length
        input_ids = input_ids_storage[:, :active_sequence_length]
        mask = mask_storage[:, :active_sequence_length]
    text_eos = embedder(torch.tensor([[talker.text_eos_token]], device=device)).to(dtype)
    text_pad = embedder(torch.tensor([[talker.text_pad_token]], device=device)).to(dtype)
    sent_text_eos = False
    av_kwargs = {}
    if not native_text_conditioning:
        n = len(prompt_tape)
        grids = torch.tensor([[int((p[0] == model.vlm_model.config.video_token_id).sum()) // 64,16,16]
                             for p in prompt_tape], device=device)
        assert n and bool((grids[:,0] > 0).all())
        av_kwargs = dict(video_grid_thw=grids, audio_feature_lengths=torch.full((n,),60,device=device),
                         video_second_per_grid=torch.full((n,), .25, device=device), use_audio_in_video=False)
    processors = LogitsProcessorList([RepetitionPenaltyLogitsProcessor(1.05),
        SuppressTokensLogitsProcessor([talker.codec_bos_token], device=device)])
    if do_sample:
        processors.extend([TemperatureLogitsWarper(.9), TopKLogitsWarper(40), TopPLogitsWarper(.8)])
    codec_rng = torch.Generator(device=device).manual_seed(seed)
    # RepetitionPenaltyLogitsProcessor depends only on whether a token has
    # occurred, not its count.  Keep that state incrementally for the isolated
    # fast-sampling A/B instead of gathering the full (often very long) AV
    # prompt at every codec step.
    seen_codec_tokens = None
    if fast_sampling:
        seen_codec_tokens = torch.zeros(
            (input_ids.shape[0], talker.codebook_size), dtype=torch.bool, device=device)
        seen_codec_tokens.scatter_(1, input_ids, True)
    pipeline = None
    def make_decoder():
        return IncrementalCode2Wav(native.token2wav,
            speaker_params['cond'].to(decoder_device),
            speaker_params['ref_mel'].to(decoder_device),
            seed=seed, geometry=geometry, solver=solver, num_steps=num_steps,
            trace=trace, first_chunk_codes=first_chunk_codes)
    if parallel_decoder and not code_only:
        decoder_stream = torch.cuda.Stream(device=decoder_device)
        same_device = decoder_device == torch.device(device)
        ready = torch.cuda.Event() if same_device else None
        if ready is not None:
            ready.record(torch.cuda.current_stream(device))
        @contextmanager
        def decoder_context():
            with torch.cuda.device(decoder_device), torch.cuda.stream(decoder_stream), \
                    torch.inference_mode(), torch.autocast('cuda', enabled=False):
                if ready is not None:
                    decoder_stream.wait_event(ready)
                yield
        pipeline = BufferedPCMDecoder(make_decoder, on_pcm, geometry=geometry,
                                      context_factory=decoder_context, trace=trace,
                                      first_chunk_codes=first_chunk_codes)
        decoder = pipeline
    else:
        decoder = None if code_only else make_decoder()
    try:
        if talker_cache == 'static':
            from transformers.cache_utils import StaticCache
            static_cache_len = input_ids.shape[1] + max_codec_tokens
            past = StaticCache(
                config=talker.config,
                max_cache_len=static_cache_len,
            )
            # Keep the compiled incremental graph shape-stable. The native
            # prefill below still uses the exact short mask; only subsequent
            # one-token calls receive this fixed-capacity causal-valid mask.
            static_mask = torch.zeros(
                (mask.shape[0], static_cache_len),
                dtype=mask.dtype,
                device=mask.device,
            )
            static_mask[:, :mask.shape[1]] = mask
        elif talker_cache == 'dynamic':
            past = None
        else:
            raise ValueError(f'Unknown Talker cache: {talker_cache}')
        if compile_talker:
            if talker_cache != 'static':
                raise ValueError('Compiled Talker requires a static KV cache')
            from transformers.generation.configuration_utils import CompileConfig
            incremental_talker_model = talker.model.get_compiled_call(
                CompileConfig(dynamic=compile_dynamic, fullgraph=False,
                              mode='default'))
        else:
            incremental_talker_model = talker.model
        new_ids = []
        pending_codec_tokens = []
        codec_eos = False
        first_codec_time = None
        codec_done_time = None
        for step in range(max_codec_tokens):
            with torch.autocast('cuda', dtype=torch.bfloat16):
                if step == 0:
                    if trace is not None:
                        trace.mark_once('talker_start')
                    prefill_cache = past if talker_cache == 'static' else None
                    out = talker(input_ids=input_ids, input_text_ids=input_text_ids,
                        attention_mask=mask, inputs_embeds=inputs_embeds.clone(),
                        cache_position=torch.arange(input_ids.shape[1], device=device),
                        past_key_values=prefill_cache,
                        use_cache=True, return_dict=True, **av_kwargs)
                    if fused_flash_kv:
                        past = FusedFlashTalkerKV(
                            out.past_key_values,
                            max_length=input_ids.shape[1] + max_codec_tokens)
                        out.past_key_values = past
                else:
                    piece = answer.pop()
                    if piece is not None:
                        feature = _hidden_map(piece[2], delta).to(dtype) + piece[1].to(dtype)
                    elif not sent_text_eos:
                        feature = text_eos
                        sent_text_eos = True
                    else:
                        feature = text_pad
                    cache_position = torch.tensor([past.get_seq_length()], device=device)
                    if fused_flash_kv:
                        codec_embed = talker.get_input_embeddings()(input_ids[:, -1:])
                        lm_input = talker.thinker_to_talker_proj(
                            codec_embed + feature[:, :1, :])
                        cache_position = torch.tensor([past.get_seq_length()], device=device)
                        delta_position = cache_position[0] + talker.rope_deltas
                        position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1)
                        hidden = past.step(talker, lm_input, position_ids)
                        class _TalkerStep:
                            pass
                        out = _TalkerStep()
                        out.logits = talker.codec_head(hidden).float()
                        out.past_key_values = past
                    elif talker_cache == 'static' or fast_talker_step:
                        # HF's Qwen2.5-Omni Talker wrapper does not forward its
                        # cache_position argument into talker.model. DynamicCache
                        # hides that omission, while StaticCache would overwrite
                        # slot zero. Reproduce only the native one-token branch
                        # here and pass the position explicitly.
                        codec_embed = talker.get_input_embeddings()(input_ids[:, -1:])
                        lm_input = talker.thinker_to_talker_proj(
                            codec_embed + feature[:, :1, :])
                        if talker_cache == 'static':
                            static_mask[:, cache_position[0]] = 1
                            step_attention_mask = static_mask
                        else:
                            # The native mask is all ones. FlashAttention turns
                            # it into no explicit mask for one-token causal
                            # decoding, so passing None is mathematically
                            # equivalent and avoids rebuilding it every step.
                            step_attention_mask = None
                        delta_position = cache_position[0] + talker.rope_deltas
                        position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1)
                        if compile_talker:
                            torch.compiler.cudagraph_mark_step_begin()
                        model_out = incremental_talker_model(
                            attention_mask=step_attention_mask,
                            position_ids=position_ids,
                            past_key_values=past,
                            inputs_embeds=lm_input,
                            use_cache=True,
                            cache_position=cache_position,
                            return_dict=True,
                        )
                        class _TalkerStep:
                            pass
                        out = _TalkerStep()
                        out.logits = talker.codec_head(model_out.last_hidden_state).float()
                        out.past_key_values = model_out.past_key_values
                    else:
                        out = talker(input_ids=input_ids[:, -1:], input_text_ids=input_text_ids,
                            attention_mask=mask, past_key_values=past, thinker_reply_part=feature,
                            cache_position=cache_position,
                            use_cache=True, return_dict=True, **av_kwargs)
                if fast_sampling:
                    scores = out.logits[:, -1].float()
                    penalty_scores = torch.where(scores < 0, scores * 1.05, scores / 1.05)
                    scores = torch.where(seen_codec_tokens, penalty_scores, scores)
                    scores[:, talker.codec_bos_token] = -float('inf')
                    if do_sample:
                        scores = scores / .9
                        top_values, _ = torch.topk(scores, 40)
                        scores = scores.masked_fill(scores < top_values[:, -1:], -float('inf'))
                        sorted_logits, sorted_indices = torch.sort(scores, descending=False)
                        remove = sorted_logits.softmax(dim=-1).cumsum(dim=-1) <= .2
                        remove[:, -1:] = False
                        remove = remove.scatter(1, sorted_indices, remove)
                        scores = scores.masked_fill(remove, -float('inf'))
                else:
                    scores = processors(input_ids, out.logits[:, -1].float())
                token = (torch.multinomial(scores.softmax(-1), 1, generator=codec_rng)
                         if do_sample else scores.argmax(-1, keepdim=True))
            past = out.past_key_values
            if seen_codec_tokens is not None:
                seen_codec_tokens.scatter_(1, token, True)
            pending_codec_tokens.append(token)
            if preallocate_sequence:
                input_ids_storage[:, active_sequence_length:active_sequence_length + 1].copy_(token)
                mask_storage[:, active_sequence_length:active_sequence_length + 1].fill_(1)
                active_sequence_length += 1
                input_ids = input_ids_storage[:, :active_sequence_length]
                mask = mask_storage[:, :active_sequence_length]
            else:
                input_ids = torch.cat((input_ids, token), 1)
                mask = torch.cat((mask, torch.ones_like(token)), 1)

            # A per-token ``token.item()`` synchronizes the entire CUDA stream.
            # It is unnecessary: the Talker consumes the GPU token directly.
            # Copy a short group only when the PCM worker needs integer codec
            # IDs.  Tokens sampled after EOS in the same group are discarded;
            # the retained sequence up to and including EOS is unchanged.
            flush_codec = (len(pending_codec_tokens) >= codec_sync_interval or
                           step + 1 == max_codec_tokens)
            if not flush_codec:
                continue
            values = torch.cat(pending_codec_tokens, dim=1)[0].tolist()
            pending_codec_tokens.clear()
            if first_codec_time is None:
                first_codec_time = time.monotonic()
                if trace is not None:
                    trace.mark_once('talker_first_codec')
            for value in values:
                new_ids.append(value)
                if value in (8292,8294):
                    codec_eos = True
                    codec_done_time = time.monotonic()
                    if trace is not None:
                        trace.mark_once('talker_eos', time.perf_counter_ns())
                    break
                if pipeline is not None:
                    pipeline.push(value)
                elif decoder is not None:
                    with torch.autocast('cuda', enabled=False):
                        for pcm,event in decoder.push([value]):
                            event['codec_eos_observed'] = False
                            on_pcm(pcm, event)
            if codec_eos:
                break
        if codec_done_time is None:
            codec_done_time = time.monotonic()
        if pipeline is not None:
            pipeline.mark_codec_finished(codec_eos)
        # Finish Thinker if Talker stopped too early; record this outcome, don't hide it.
        text_finished_before_codec = answer.done
        with torch.autocast('cuda', dtype=torch.bfloat16):
            while not answer.done:
                answer.pop()
        if pipeline is not None:
            pipeline.finish()
        elif decoder is not None:
            with torch.autocast('cuda', enabled=False):
                for pcm,event in decoder.finish():
                    event['codec_eos_observed'] = codec_eos
                    on_pcm(pcm,event)
        text_result = answer.result()
        if trace is not None:
            trace.mark_once('native_thinker_end')
            trace.set('generated_text_tokens', len(text_result.get('ids', [])))
            trace.set('generated_codec_count', len(new_ids))
            trace.set('first_chunk_codec_count', int(first_chunk_codes or
                      (geometry.chunk_codes if geometry else SpeechGeometry().chunk_codes)))
        return dict(native_speech=decoder is not None, **text_result,
            speaker=speaker, codec_ids=new_ids, codec_tokens=len(new_ids), talker_eos=codec_eos,
            first_codec_monotonic=first_codec_time, codec_complete_monotonic=codec_done_time,
            decode_speech_seconds=time.monotonic()-before,
            text_finished_before_codec=text_finished_before_codec,
            solver=solver, num_steps=num_steps, code_only=code_only,
            decoder_pipeline='parallel' if pipeline is not None else 'serial',
            decoder_device=str(decoder_device),
            codec_sync_interval=int(codec_sync_interval),
            preallocate_sequence=bool(preallocate_sequence),
            fast_talker_step=bool(fast_talker_step),
            fast_sampling=bool(fast_sampling),
            fused_flash_kv=bool(fused_flash_kv),
            decoder_max_pending=0 if pipeline is None else pipeline.max_pending_observed,
            decoder_backpressure_seconds=0. if pipeline is None else pipeline.backpressure_seconds,
            audio_chunks=[] if decoder is None else decoder.events,
            native_streaming_scope='incremental text + native codec + native PCM; concurrent body/input requires separate worker',
            samples=0 if decoder is None else decoder.emitted_samples)
    except BaseException as error:
        if trace is not None:
            trace.set('cancelled', True)
            trace.set('cancel_reason', type(error).__name__)
            trace.mark_once('native_thinker_end')
        raise
    finally:
        if pipeline is not None:
            pipeline.close()