"""Native incremental speech primitives retained for the formal HFD runtime. Native Qwen2.5-Omni weights are reused. There is no external TTS and no whole-utterance waveform to slice. A stateful code->mel->PCM decoder accepts new codec tokens, emits committed audio, and retains overlap CONTEXT only. The window geometry is adapted from QwenLM/Qwen2.5-Omni (Apache-2.0), commit d8a31ca56c0456b6edfcbcbf4bdbb6ae2200ef42, low-VRAM-mode. Copyright 2025 Qwen Team / HuggingFace Inc. Unlike that example we expose PCM as it is made, never delete model modules, and retain the installed 4.57 RK4 solver by default. The formal default is selected by :class:`psi.runtime.speech.SpeechRuntimeConfig`: Euler-6, 36 codec codes and a 0.20-second playback threshold. The module is optional; importing ``psi.runtime`` does not import model-specific helpers. """ from dataclasses import dataclass, replace from collections import deque from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager, nullcontext import threading import time import torch from transformers.cache_utils import DynamicCache REPLY_QUERY = ( '<|im_end|>\n<|im_start|>user\nAnswer the latest spoken visual question ' 'with a short phrase. Use the camera view when it was asked.<|im_end|>\n' '<|im_start|>assistant\n' ) def _clone_selected_cache(past, rows): return DynamicCache(ddp_cache_data=[ (layer.keys.index_select(0, rows), layer.values.index_select(0, rows)) for layer in past.layers ]) @dataclass(frozen=True) class SpeechGeometry: chunk_codes: int = 48 left_codes: int = 24 right_codes: int = 12 vocoder_left: int = 20 vocoder_right: int = 20 sample_rate: int = 24000 hop: int = 240 max_mel_frames: int = 30000 class PCMClockPackets: """Reblock already-produced audio onto the action clock, never a full WAV. At 24 kHz, 9,600 NEW samples correspond to one 0.4 s action step. The speech decoder may have a different native block size; this small FIFO reconciles the clocks without resetting its hidden/context states. """ def __init__(self, samples_per_packet=9600): assert samples_per_packet > 0 self.size = samples_per_packet self.buffer = torch.empty(0, dtype=torch.float32) self.received = 0 self.emitted = 0 self.closed = False self.last_event = None def push(self, pcm, event): if self.closed: raise RuntimeError('PCM clock already closed') assert pcm.device.type == 'cpu' and pcm.ndim == 1 assert event['sample_start'] == self.received self.received += pcm.numel() assert event['sample_end'] == self.received self.last_event = dict(event) self.buffer = torch.cat((self.buffer, pcm.float())) result = [] while self.buffer.numel() >= self.size: data = self.buffer[:self.size].clone() self.buffer = self.buffer[self.size:] row = dict(event, sample_start=self.emitted, samples=data.numel(), sample_end=self.emitted + data.numel(), available_monotonic=time.monotonic(), partial_final=False) self.emitted += data.numel() result.append((data, row)) return result def finish(self): if self.closed: raise RuntimeError('PCM clock already closed') self.closed = True result = [] if self.buffer.numel(): row = dict(self.last_event, sample_start=self.emitted, samples=self.buffer.numel(), sample_end=self.received, available_monotonic=time.monotonic(), partial_final=True) result.append((self.buffer.clone(), row)) self.emitted = self.received assert self.emitted == self.received self.buffer = self.buffer[:0] return result class IncrementalCode2Wav: """One utterance, B=1. push/finish return NEW, nonoverlapping PCM chunks. Code context is not reset at each chunk; noise uses global mel coordinates. No cross-fade is applied to conceal decoder errors. Boundary quality must be measured and listened to. This class does not itself enforce wall pacing. """ def __init__(self, token2wav, conditioning, reference_mel, *, seed, geometry=None, solver='rk4', num_steps=10, guidance_scale=.5, sway_coefficient=-1., sample_fn=None, trace=None, first_chunk_codes=None): self.geometry = geometry or SpeechGeometry() self.regular_geometry = self.geometry first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None else int(first_chunk_codes)) self.first_geometry = replace(self.geometry, chunk_codes=first_chunk_codes) self.dit = token2wav.code2wav_dit_model self.vocoder = token2wav.code2wav_bigvgan_model self.conditioning = conditioning.float() self.reference = reference_mel.float() self.device = self.reference.device self.solver = solver self.num_steps = num_steps self.guidance_scale = guidance_scale self.sway = sway_coefficient self.sample_fn = sample_fn self.trace = trace self.repeats = int(self.dit.repeats) g = self.geometry assert self.reference.shape[0] == self.conditioning.shape[0] == 1 assert g.chunk_codes > 0 and g.left_codes >= 0 and g.right_codes >= 0 # Global block alignment is essential for the native block-attention mask. assert (g.chunk_codes * self.repeats) % self.dit.block_size == 0 assert (g.left_codes * self.repeats) % self.dit.block_size == 0 assert (self.first_geometry.chunk_codes * self.repeats) % self.dit.block_size == 0 assert solver in ('rk4', 'euler') and num_steps >= 2 assert g.vocoder_left >= 0 and g.vocoder_right >= 0 generator = torch.Generator(device='cpu').manual_seed(seed) # Generate once on CPU for deterministic compatibility with the old # implementation, then move the complete noise bank to the decoder # device. The previous code copied one slice CPU->GPU on every # Token2Wav block, which put a host transfer on the critical path. noise = torch.randn(1, g.max_mel_frames, self.dit.mel_dim, generator=generator, dtype=self.reference.dtype) self.noise = noise.to(self.device) del noise self.code = torch.empty(1, 0, device=self.device, dtype=torch.long) self.mel = torch.empty(1, self.dit.mel_dim, 0, device=self.device) self.code_left = 0 self.mel_left = 0 self.code_start = 0 self.received = 0 self.committed_codes = 0 self.committed_mel = 0 self.emitted_samples = 0 self.finished = False self.events = [] def _sample(self, code, start): if self.trace is not None: self.trace.mark_once('token2wav_first_start') count = code.shape[1] * self.repeats if start < 0 or start + count > self.noise.shape[1]: raise ValueError('Utterance exceeds explicit noise/context capacity') initial = self.noise[:, start:start + count] if self.sample_fn is not None: result = self.sample_fn(code, initial, start) if self.trace is not None: self.trace.mark_once('token2wav_first_end') return result condition = self.conditioning[:, None].expand(-1, count, -1) def velocity(t, state): output = self.dit(hidden_states=state, quantized_code=code, speaker_embedding=condition, condition_vector=self.reference, time_step=t, apply_cfg=self.guidance_scale >= 1e-5) if self.guidance_scale < 1e-5: return output guided, null = output.chunk(2, dim=0) return guided + (guided - null) * self.guidance_scale times = torch.linspace(0, 1, self.num_steps, device=self.device, dtype=self.reference.dtype) if self.sway is not None: times += self.sway * (torch.cos(torch.pi / 2 * times) - 1 + times) if self.solver == 'rk4': from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import RungeKutta4ODESolver result = RungeKutta4ODESolver(velocity, initial).integrate(times)[-1] else: result = initial for t0, t1 in zip(times[:-1], times[1:]): result = result + velocity(t0, result) * (t1 - t0) result = result.transpose(1, 2) if self.trace is not None: self.trace.mark_once('token2wav_first_end') return result def _vocode(self, new_mel, *, final=False): g = self.geometry self.mel = torch.cat((self.mel, new_mel), dim=-1) right = 0 if final else g.vocoder_right frames = self.mel.shape[-1] if frames <= self.mel_left + right: return [] wave = self.vocoder(self.mel).reshape(-1) assert wave.numel() == frames * g.hop, 'Unexpected native vocoder hop' end = wave.numel() - right * g.hop pcm = wave[self.mel_left * g.hop:end].float().detach().cpu().clone() assert pcm.numel() and torch.isfinite(pcm).all() if self.trace is not None: # This is the first point at which Token2Wav has produced actual # PCM samples. The caller may still need to write/enqueue them. self.trace.mark_once('talker_first_chunk_ready') # .cpu() is intentionally a completion boundary: these samples are ready # to play now, not merely queued CUDA operations with premature timestamps. event = dict(index=len(self.events), codec_received=self.received, code_prefix_committed=self.committed_codes, sample_start=self.emitted_samples, samples=pcm.numel(), sample_end=self.emitted_samples + pcm.numel(), final=final, available_monotonic=time.monotonic()) self.events.append(event) self.emitted_samples += pcm.numel() if not final: keep = min(frames, g.vocoder_left + g.vocoder_right) self.mel = self.mel[:, :, -keep:] if keep else self.mel[:, :, :0] self.mel_left = keep - g.vocoder_right assert self.mel_left >= 0 else: self.mel = self.mel[:, :, :0] return [(pcm, event)] @torch.inference_mode() def push(self, tokens): if self.finished: raise RuntimeError('Cannot append to a finished speech stream') values = torch.as_tensor(tokens, dtype=torch.long, device=self.device).reshape(-1) output = [] for token in values: g = (self.first_geometry if self.committed_codes == 0 else self.regular_geometry) self.geometry = g self.code = torch.cat((self.code, token.reshape(1, 1)), dim=1) self.received += 1 if self.received * self.repeats > g.max_mel_frames: raise ValueError('Utterance exceeds explicit noise/context capacity') target = self.code_left + g.chunk_codes + g.right_codes if self.code.shape[1] < target: continue assert self.code.shape[1] == target mel = self._sample(self.code, self.code_start * self.repeats) first = self.code_left * self.repeats last = (self.code.shape[1] - g.right_codes) * self.repeats new = mel[:, :, first:last] assert new.shape[-1] == g.chunk_codes * self.repeats self.committed_codes += g.chunk_codes self.committed_mel += new.shape[-1] output.extend(self._vocode(new)) keep = min(self.code.shape[1], g.left_codes + g.right_codes) self.code = self.code[:, -keep:] if keep else self.code[:, :0] self.code_left = keep - g.right_codes self.code_start = self.committed_codes - self.code_left self.geometry = self.regular_geometry return output @torch.inference_mode() def finish(self): if self.finished: raise RuntimeError('Speech stream already finished') self.finished = True remaining = self.received - self.committed_codes output = [] if remaining: mel = self._sample(self.code, self.code_start * self.repeats) new = mel[:, :, self.code_left * self.repeats:] assert new.shape[-1] == remaining * self.repeats self.committed_codes += remaining self.committed_mel += new.shape[-1] # Flush all vocoder right context exactly once at utterance end. output.extend(self._vocode(new, final=True)) elif self.mel.shape[-1] > self.mel_left: empty = self.mel[:, :, :0] output.extend(self._vocode(empty, final=True)) assert self.committed_codes == self.received assert self.emitted_samples == self.received * self.repeats * self.geometry.hop self.code = self.code[:, :0] return output class BufferedPCMDecoder: """Bounded, ordered codec->PCM worker; no future tokens or full WAV. The producer submits only newly available native chunks. One decoder owns all context/noise state in one worker thread. At most two jobs may be outstanding; backpressure never drops codes. Callback completion is joined before close returns, including on producer/decoder failure. """ def __init__(self, factory, on_pcm, *, geometry=None, context_factory=nullcontext, max_pending=2, trace=None, first_chunk_codes=None): self.geometry = geometry or SpeechGeometry() if max_pending < 1: raise ValueError('At least one outstanding decoder job is required') self.factory, self.on_pcm = factory, on_pcm self.context_factory = context_factory self.max_pending = max_pending self.trace = trace self.pending = deque() self.buffer = [] self.first_chunk_codes = (self.geometry.chunk_codes if first_chunk_codes is None else int(first_chunk_codes)) self.target = self.first_chunk_codes + self.geometry.right_codes self.executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='native-pcm') self.decoder = None self.failure = None self.failure_observed = False self.finished = False self.closed = False self.producer_done = threading.Event() self.codec_eos = False self.max_pending_observed = 0 self.backpressure_seconds = 0. def mark_codec_finished(self, eos): self.codec_eos = bool(eos) self.producer_done.set() def _run(self, values, final): if self.failure is not None: raise RuntimeError('Decoder pipeline failed on an earlier chunk') from self.failure try: with self.context_factory(): if self.decoder is None: self.decoder = self.factory() output = self.decoder.push(values) if final: output.extend(self.decoder.finish()) for pcm, event in output: # In a pipeline the producer may reach EOS while this worker # decodes an earlier chunk. Report actual completion, not the # state when the chunk was enqueued. event['codec_eos_observed'] = self.producer_done.is_set() and self.codec_eos event['codec_generation_finished'] = self.producer_done.is_set() self.on_pcm(pcm, event) except BaseException as error: self.failure = error raise def _result(self, future): try: return future.result() except BaseException: self.failure_observed = True raise def _submit(self, values, final=False): while self.pending and self.pending[0].done(): self._result(self.pending.popleft()) if len(self.pending) >= self.max_pending: start = time.monotonic() self._result(self.pending.popleft()) self.backpressure_seconds += time.monotonic() - start self.pending.append(self.executor.submit(self._run, tuple(values), final)) self.max_pending_observed = max(self.max_pending_observed, len(self.pending)) if self.trace is not None: self.trace.set('speech_queue_depth', len(self.pending)) self.trace.set('speech_queue_max_depth', self.max_pending_observed) def push(self, value): if self.closed or self.finished: raise RuntimeError('Cannot append to a closed decoder pipeline') self.buffer.append(int(value)) if len(self.buffer) == self.target: self._submit(self.buffer) self.buffer = [] self.target = self.geometry.chunk_codes def finish(self): if self.closed or self.finished: raise RuntimeError('Decoder pipeline already finished') self.finished = True self._submit(self.buffer, final=True) self.buffer = [] while self.pending: self._result(self.pending.popleft()) @property def events(self): return [] if self.decoder is None else self.decoder.events @property def emitted_samples(self): return 0 if self.decoder is None else self.decoder.emitted_samples def close(self): if not self.closed: self.closed = True # On failure cancel only jobs not started yet; join the running # callback before its caller closes the audio sink or model. self.executor.shutdown(wait=True, cancel_futures=True) if self.failure is not None and not self.failure_observed: self.failure_observed = True # If the producer also failed, Python retains it as __context__ in # the traceback. Do not silently hide a concurrent audio failure. raise self.failure class IncrementalAnswer: """Greedy Thinker side branch; one causal text feature returned per pop(). No GT text and no future camera/audio enter this branch. The input history is cloned, so ongoing perception/action may continue appending its own KV. """ @torch.inference_mode() def __init__(self, model, cache, max_tokens=48): self.model = model self.thinker = model.vlm_model self.device = next(model.reply_head.parameters()).device self.max_tokens = max_tokens self.generated = [] self.tape = [] self.done = False self.eos = False self.complete_monotonic = None self.source_cache = cache self.source_length = cache.token_count self.past = _clone_selected_cache(cache.past_key_values, torch.tensor([0], device=self.device)) ids = torch.tensor([model.vlm_processor.tokenizer.encode( REPLY_QUERY, add_special_tokens=False)], device=self.device) n = ids.shape[1] self.offset = cache.av_next_positions[0] + n if cache.token_count + n + max_tokens > model.av_context_limit: raise ValueError('Incremental answer exceeds context capacity') self.mask = torch.cat((cache.attention_mask[:1], torch.ones_like(ids)), 1) emb = self.thinker.get_input_embeddings()(ids) self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask, position_ids=(torch.arange(n, device=self.device) + cache.av_next_positions[0])[None,None].expand(3,1,-1), past_key_values=self.past, cache_position=torch.arange(cache.token_count, cache.token_count+n, device=self.device), use_cache=True, return_dict=True) self.past = self.out.past_key_values self.query = (ids, emb, self.out.last_hidden_state) @torch.inference_mode() def pop(self): if self.done: return None token = self.thinker.lm_head(self.out.last_hidden_state[:, -1:]).argmax(-1) value = int(token.item()) self.generated.append(value) self.eos = value == int(self.thinker.config.eos_token_id) if self.eos: self.done = True self.complete_monotonic = time.monotonic() return None emb = self.thinker.get_input_embeddings()(token) count = self.past.get_seq_length() self.mask = torch.cat((self.mask, torch.ones_like(token)), 1) self.out = self.thinker.model(inputs_embeds=emb, attention_mask=self.mask, past_key_values=self.past, position_ids=torch.full((3,1,1), self.offset, device=self.device, dtype=torch.long), cache_position=torch.tensor([count], device=self.device), use_cache=True, return_dict=True) self.past = self.out.past_key_values self.offset += 1 piece = (token, emb, self.out.last_hidden_state) self.tape.append(piece) if len(self.generated) >= self.max_tokens: self.done = True self.complete_monotonic = time.monotonic() return piece def result(self): assert self.done assert self.source_cache.past_key_values.get_seq_length() == self.source_length return dict(text=self.model.vlm_processor.tokenizer.decode(self.generated, skip_special_tokens=True), ids=self.generated, eos=self.eos, branch_tape=[self.query] + self.tape, text_complete_monotonic=self.complete_monotonic) class NativeTextConditionAnswer: """The trained AV model chooses words; frozen native features drive speech. Only tokens already emitted by the trained model enter the frozen voice KV. No GT answer, future input, or trained hidden-state distribution is passed to the frozen Talker. Both private caches leave the action cache untouched. """ @torch.inference_mode() def __init__(self, model, cache, voice_thinker, max_tokens=48): self.learner = IncrementalAnswer(model, cache, max_tokens) self.voice = voice_thinker self.device = self.learner.device tokenizer = model.vlm_processor.tokenizer ids = torch.tensor([tokenizer.apply_chat_template([ dict(role='system', content='You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'), dict(role='user', content='Give a short spoken answer.')], tokenize=True, add_generation_prompt=True)], device=self.device) self.mask = torch.ones_like(ids) emb = self.voice.get_input_embeddings()(ids) self.offset = ids.shape[1] out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask, position_ids=torch.arange(self.offset, device=self.device)[None, None].expand(3, 1, -1), cache_position=torch.arange(self.offset, device=self.device), use_cache=True, return_dict=True) self.past = out.past_key_values self.query = (ids, emb, out.last_hidden_state) self.arrived_ids = [] @property def done(self): return self.learner.done @torch.inference_mode() def pop(self): piece = self.learner.pop() if piece is None: return None ids = piece[0] self.arrived_ids.extend(ids[0].tolist()) emb = self.voice.get_input_embeddings()(ids) self.mask = torch.cat((self.mask, torch.ones_like(ids)), 1) count = self.past.get_seq_length() out = self.voice.model(inputs_embeds=emb, attention_mask=self.mask, position_ids=torch.full((3, 1, 1), self.offset, device=self.device, dtype=torch.long), cache_position=torch.tensor([count], device=self.device), past_key_values=self.past, use_cache=True, return_dict=True) self.past = out.past_key_values self.offset += 1 return ids, emb, out.last_hidden_state def result(self): result = self.learner.result() expected = [t for t in result['ids'] if t != int(self.learner.thinker.config.eos_token_id)] if self.arrived_ids != expected: raise RuntimeError('Native voice did not consume exactly the generated answer tokens') return dict(result, voice_conditioning='frozen_base_text_causal', voice_arrived_ids=self.arrived_ids) def _hidden_map(hidden, delta): if delta is None: return hidden # Match the train-only fitted interface exactly. The caller alternates # BF16 Thinker/Talker and FP32 decoder contexts; outer autocast must NOT # silently quantize this FP32 matrix multiply on later text tokens. shape = hidden.shape with torch.autocast(device_type=hidden.device.type, enabled=False): x = hidden.float().reshape(-1, shape[-1]) mapped = x + x @ delta[:-1] + delta[-1] return mapped.reshape(shape).to(hidden.dtype) class FusedFlashTalkerKV: """Fixed-capacity Talker KV using FlashAttention's decode kernel. The native prefill remains untouched. This object only replaces the one-token DynamicCache growth path, which otherwise reallocates and concatenates every layer's full K/V history at every codec token. """ def __init__(self, dynamic_cache, *, max_length): from flash_attn import flash_attn_with_kvcache self.flash_attn_with_kvcache = flash_attn_with_kvcache self.length = int(dynamic_cache.get_seq_length()) if self.length >= max_length: raise ValueError('Fused Talker cache capacity must exceed prefill length') self.max_length = int(max_length) self.keys = [] self.values = [] for layer in dynamic_cache.layers: # HF cache: [B, Hkv, S, D]. Flash decode cache: [B, S, Hkv, D]. key = layer.keys.transpose(1, 2).contiguous() value = layer.values.transpose(1, 2).contiguous() key_store = torch.empty( (key.shape[0], self.max_length, key.shape[2], key.shape[3]), dtype=key.dtype, device=key.device) value_store = torch.empty_like(key_store) key_store[:, :self.length].copy_(key) value_store[:, :self.length].copy_(value) self.keys.append(key_store) self.values.append(value_store) self.cache_seqlens = torch.full( (self.keys[0].shape[0],), self.length, dtype=torch.int32, device=self.keys[0].device) def get_seq_length(self): return self.length def _step_impl(self, talker, hidden_states, position_ids): from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import ( apply_multimodal_rotary_pos_emb, ) position_embeddings = talker.model.rotary_emb(hidden_states, position_ids) for index, layer in enumerate(talker.model.layers): residual = hidden_states normed = layer.input_layernorm(hidden_states) attn = layer.self_attn batch, query_length, _ = normed.shape query = attn.q_proj(normed).view( batch, query_length, attn.num_heads, attn.head_dim) key = attn.k_proj(normed).view( batch, query_length, attn.num_key_value_heads, attn.head_dim) value = attn.v_proj(normed).view( batch, query_length, attn.num_key_value_heads, attn.head_dim) cos, sin = position_embeddings query, key = apply_multimodal_rotary_pos_emb( query.transpose(1, 2), key.transpose(1, 2), cos, sin, attn.rope_scaling['mrope_section']) query = query.transpose(1, 2).contiguous() key = key.transpose(1, 2).contiguous() attended = self.flash_attn_with_kvcache( query, self.keys[index], self.values[index], k=key, v=value, cache_seqlens=self.cache_seqlens, softmax_scale=attn.scaling, causal=True, num_splits=1) attended = attended.reshape(batch, query_length, -1).contiguous() hidden_states = residual + attn.o_proj(attended) residual = hidden_states hidden_states = residual + layer.mlp( layer.post_attention_layernorm(hidden_states)) return talker.model.norm(hidden_states) def step(self, talker, hidden_states, position_ids): """Run exactly one causal Talker token and append its K/V in-place.""" if hidden_states.shape[1] != 1: raise ValueError('Fused Talker decode only supports one new token') if self.length >= self.max_length: raise RuntimeError('Fused Talker KV cache is full') hidden_states = self._step_impl(talker, hidden_states, position_ids) self.length += 1 self.cache_seqlens.fill_(self.length) return hidden_states @torch.inference_mode() def stream_native_reply(native, model, cache, prompt_tape, *, seed, on_pcm, speaker='Chelsie', adapter=None, geometry=None, solver='rk4', num_steps=10, max_codec_tokens=500, code_only=False, do_sample=True, parallel_decoder=False, trace=None, decoder_device=None, first_chunk_codes=None, talker_cache='dynamic', compile_talker=False, compile_dynamic=False, codec_sync_interval=1, preallocate_sequence=False, fast_talker_step=False, fast_sampling=False, fused_flash_kv=False, native_text_conditioning=False): """Interleave Thinker tokens and native Talker tokens; callback NEW PCM. This function belongs in a separate worker/CUDA stream while the body loop runs. Its return contains metrics and codec IDs, NOT a hidden full WAV. No whole text answer is generated before Talker starts. Tiny answers may naturally finish before the decoder's finite-lookahead first audio chunk. parallel_decoder overlaps codec production and PCM decoding on independent CUDA streams, with a bounded FIFO; default False preserves the serial arm. """ from transformers.generation.logits_process import ( LogitsProcessorList, RepetitionPenaltyLogitsProcessor, SuppressTokensLogitsProcessor, TemperatureLogitsWarper, TopKLogitsWarper, TopPLogitsWarper) before = time.monotonic() if codec_sync_interval < 1: raise ValueError('codec_sync_interval must be positive') if trace is not None: trace.mark_once('speech_job_start') learner_device = model.device talker = native.talker device = next(talker.parameters()).device decoder_device = torch.device(device if decoder_device is None else decoder_device) talker.rope_deltas = None # reset per utterance, not per chunk dtype = talker.thinker_to_talker_proj.weight.dtype delta = adapter.to(device=device, dtype=torch.float32) if adapter is not None else None if trace is not None: trace.mark_once('native_thinker_start') with torch.autocast('cuda', dtype=torch.bfloat16): answer = (NativeTextConditionAnswer(model, cache, native.voice_thinker) if native_text_conditioning else IncrementalAnswer(model, cache)) first = answer.pop() if first is not None and trace is not None: trace.mark_once('native_thinker_first_hidden') if first is None: return dict(native_speech=False, native_speech_error='empty generated answer', **answer.result()) if native_text_conditioning: delta = None tape = [answer.query] if native_text_conditioning else list(prompt_tape) + [answer.query] ids = torch.cat([x[0] for x in tape], 1) emb = torch.cat([x[1] for x in tape], 1).to(dtype).clone() # Use exactly the fitted/evaluated per-packet GEMM shapes. Mapping one # concatenated history instead changes floating-point kernel/reduction # choices; BF16 rounding then perturbs autoregressive sampling later on. hidden = torch.cat([_hidden_map(x[2], delta).to(dtype) for x in tape], 1) # Same masking as native.generate: AV semantics are carried by hidden, # not reintroduced as raw frontend embeddings into Talker. for name in ('audio_token_id', 'image_token_id', 'video_token_id'): emb.masked_fill_(ids.eq(getattr(model.vlm_model.config, name))[:, :, None], 0) speaker_params = native.speaker_map[speaker] # In the split-GPU path, the frozen voice Thinker and Talker live together # while the AV learner keeps its source KV on the body GPU. The embedding # weights are identical, but using the voice copy avoids a device roundtrip. embedder = (native.voice_thinker.get_input_embeddings() if hasattr(native, 'voice_thinker') else model.vlm_model.get_input_embeddings()) text_bos = torch.tensor([[speaker_params['bos_token']]], device=device) input_text_ids = torch.cat((ids, text_bos, first[0]), 1) input_ids = torch.cat((torch.full_like(ids, talker.codec_mask_token), torch.tensor([[talker.codec_pad_token, talker.codec_bos_token]], device=device)), 1) first_feature = _hidden_map(first[2], delta).to(dtype) + first[1].to(dtype) inputs_embeds = torch.cat((hidden + emb, embedder(text_bos).to(dtype), first_feature), 1) mask = torch.ones_like(input_ids) if preallocate_sequence: initial_sequence_length = input_ids.shape[1] input_ids_storage = torch.empty( (input_ids.shape[0], initial_sequence_length + max_codec_tokens), dtype=input_ids.dtype, device=input_ids.device) mask_storage = torch.zeros_like(input_ids_storage) input_ids_storage[:, :initial_sequence_length].copy_(input_ids) mask_storage[:, :initial_sequence_length].copy_(mask) active_sequence_length = initial_sequence_length input_ids = input_ids_storage[:, :active_sequence_length] mask = mask_storage[:, :active_sequence_length] text_eos = embedder(torch.tensor([[talker.text_eos_token]], device=device)).to(dtype) text_pad = embedder(torch.tensor([[talker.text_pad_token]], device=device)).to(dtype) sent_text_eos = False av_kwargs = {} if not native_text_conditioning: n = len(prompt_tape) grids = torch.tensor([[int((p[0] == model.vlm_model.config.video_token_id).sum()) // 64,16,16] for p in prompt_tape], device=device) assert n and bool((grids[:,0] > 0).all()) av_kwargs = dict(video_grid_thw=grids, audio_feature_lengths=torch.full((n,),60,device=device), video_second_per_grid=torch.full((n,), .25, device=device), use_audio_in_video=False) processors = LogitsProcessorList([RepetitionPenaltyLogitsProcessor(1.05), SuppressTokensLogitsProcessor([talker.codec_bos_token], device=device)]) if do_sample: processors.extend([TemperatureLogitsWarper(.9), TopKLogitsWarper(40), TopPLogitsWarper(.8)]) codec_rng = torch.Generator(device=device).manual_seed(seed) # RepetitionPenaltyLogitsProcessor depends only on whether a token has # occurred, not its count. Keep that state incrementally for the isolated # fast-sampling A/B instead of gathering the full (often very long) AV # prompt at every codec step. seen_codec_tokens = None if fast_sampling: seen_codec_tokens = torch.zeros( (input_ids.shape[0], talker.codebook_size), dtype=torch.bool, device=device) seen_codec_tokens.scatter_(1, input_ids, True) pipeline = None def make_decoder(): return IncrementalCode2Wav(native.token2wav, speaker_params['cond'].to(decoder_device), speaker_params['ref_mel'].to(decoder_device), seed=seed, geometry=geometry, solver=solver, num_steps=num_steps, trace=trace, first_chunk_codes=first_chunk_codes) if parallel_decoder and not code_only: decoder_stream = torch.cuda.Stream(device=decoder_device) same_device = decoder_device == torch.device(device) ready = torch.cuda.Event() if same_device else None if ready is not None: ready.record(torch.cuda.current_stream(device)) @contextmanager def decoder_context(): with torch.cuda.device(decoder_device), torch.cuda.stream(decoder_stream), \ torch.inference_mode(), torch.autocast('cuda', enabled=False): if ready is not None: decoder_stream.wait_event(ready) yield pipeline = BufferedPCMDecoder(make_decoder, on_pcm, geometry=geometry, context_factory=decoder_context, trace=trace, first_chunk_codes=first_chunk_codes) decoder = pipeline else: decoder = None if code_only else make_decoder() try: if talker_cache == 'static': from transformers.cache_utils import StaticCache static_cache_len = input_ids.shape[1] + max_codec_tokens past = StaticCache( config=talker.config, max_cache_len=static_cache_len, ) # Keep the compiled incremental graph shape-stable. The native # prefill below still uses the exact short mask; only subsequent # one-token calls receive this fixed-capacity causal-valid mask. static_mask = torch.zeros( (mask.shape[0], static_cache_len), dtype=mask.dtype, device=mask.device, ) static_mask[:, :mask.shape[1]] = mask elif talker_cache == 'dynamic': past = None else: raise ValueError(f'Unknown Talker cache: {talker_cache}') if compile_talker: if talker_cache != 'static': raise ValueError('Compiled Talker requires a static KV cache') from transformers.generation.configuration_utils import CompileConfig incremental_talker_model = talker.model.get_compiled_call( CompileConfig(dynamic=compile_dynamic, fullgraph=False, mode='default')) else: incremental_talker_model = talker.model new_ids = [] pending_codec_tokens = [] codec_eos = False first_codec_time = None codec_done_time = None for step in range(max_codec_tokens): with torch.autocast('cuda', dtype=torch.bfloat16): if step == 0: if trace is not None: trace.mark_once('talker_start') prefill_cache = past if talker_cache == 'static' else None out = talker(input_ids=input_ids, input_text_ids=input_text_ids, attention_mask=mask, inputs_embeds=inputs_embeds.clone(), cache_position=torch.arange(input_ids.shape[1], device=device), past_key_values=prefill_cache, use_cache=True, return_dict=True, **av_kwargs) if fused_flash_kv: past = FusedFlashTalkerKV( out.past_key_values, max_length=input_ids.shape[1] + max_codec_tokens) out.past_key_values = past else: piece = answer.pop() if piece is not None: feature = _hidden_map(piece[2], delta).to(dtype) + piece[1].to(dtype) elif not sent_text_eos: feature = text_eos sent_text_eos = True else: feature = text_pad cache_position = torch.tensor([past.get_seq_length()], device=device) if fused_flash_kv: codec_embed = talker.get_input_embeddings()(input_ids[:, -1:]) lm_input = talker.thinker_to_talker_proj( codec_embed + feature[:, :1, :]) cache_position = torch.tensor([past.get_seq_length()], device=device) delta_position = cache_position[0] + talker.rope_deltas position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1) hidden = past.step(talker, lm_input, position_ids) class _TalkerStep: pass out = _TalkerStep() out.logits = talker.codec_head(hidden).float() out.past_key_values = past elif talker_cache == 'static' or fast_talker_step: # HF's Qwen2.5-Omni Talker wrapper does not forward its # cache_position argument into talker.model. DynamicCache # hides that omission, while StaticCache would overwrite # slot zero. Reproduce only the native one-token branch # here and pass the position explicitly. codec_embed = talker.get_input_embeddings()(input_ids[:, -1:]) lm_input = talker.thinker_to_talker_proj( codec_embed + feature[:, :1, :]) if talker_cache == 'static': static_mask[:, cache_position[0]] = 1 step_attention_mask = static_mask else: # The native mask is all ones. FlashAttention turns # it into no explicit mask for one-token causal # decoding, so passing None is mathematically # equivalent and avoids rebuilding it every step. step_attention_mask = None delta_position = cache_position[0] + talker.rope_deltas position_ids = delta_position.reshape(1, 1, 1).expand(3, 1, 1) if compile_talker: torch.compiler.cudagraph_mark_step_begin() model_out = incremental_talker_model( attention_mask=step_attention_mask, position_ids=position_ids, past_key_values=past, inputs_embeds=lm_input, use_cache=True, cache_position=cache_position, return_dict=True, ) class _TalkerStep: pass out = _TalkerStep() out.logits = talker.codec_head(model_out.last_hidden_state).float() out.past_key_values = model_out.past_key_values else: out = talker(input_ids=input_ids[:, -1:], input_text_ids=input_text_ids, attention_mask=mask, past_key_values=past, thinker_reply_part=feature, cache_position=cache_position, use_cache=True, return_dict=True, **av_kwargs) if fast_sampling: scores = out.logits[:, -1].float() penalty_scores = torch.where(scores < 0, scores * 1.05, scores / 1.05) scores = torch.where(seen_codec_tokens, penalty_scores, scores) scores[:, talker.codec_bos_token] = -float('inf') if do_sample: scores = scores / .9 top_values, _ = torch.topk(scores, 40) scores = scores.masked_fill(scores < top_values[:, -1:], -float('inf')) sorted_logits, sorted_indices = torch.sort(scores, descending=False) remove = sorted_logits.softmax(dim=-1).cumsum(dim=-1) <= .2 remove[:, -1:] = False remove = remove.scatter(1, sorted_indices, remove) scores = scores.masked_fill(remove, -float('inf')) else: scores = processors(input_ids, out.logits[:, -1].float()) token = (torch.multinomial(scores.softmax(-1), 1, generator=codec_rng) if do_sample else scores.argmax(-1, keepdim=True)) past = out.past_key_values if seen_codec_tokens is not None: seen_codec_tokens.scatter_(1, token, True) pending_codec_tokens.append(token) if preallocate_sequence: input_ids_storage[:, active_sequence_length:active_sequence_length + 1].copy_(token) mask_storage[:, active_sequence_length:active_sequence_length + 1].fill_(1) active_sequence_length += 1 input_ids = input_ids_storage[:, :active_sequence_length] mask = mask_storage[:, :active_sequence_length] else: input_ids = torch.cat((input_ids, token), 1) mask = torch.cat((mask, torch.ones_like(token)), 1) # A per-token ``token.item()`` synchronizes the entire CUDA stream. # It is unnecessary: the Talker consumes the GPU token directly. # Copy a short group only when the PCM worker needs integer codec # IDs. Tokens sampled after EOS in the same group are discarded; # the retained sequence up to and including EOS is unchanged. flush_codec = (len(pending_codec_tokens) >= codec_sync_interval or step + 1 == max_codec_tokens) if not flush_codec: continue values = torch.cat(pending_codec_tokens, dim=1)[0].tolist() pending_codec_tokens.clear() if first_codec_time is None: first_codec_time = time.monotonic() if trace is not None: trace.mark_once('talker_first_codec') for value in values: new_ids.append(value) if value in (8292,8294): codec_eos = True codec_done_time = time.monotonic() if trace is not None: trace.mark_once('talker_eos', time.perf_counter_ns()) break if pipeline is not None: pipeline.push(value) elif decoder is not None: with torch.autocast('cuda', enabled=False): for pcm,event in decoder.push([value]): event['codec_eos_observed'] = False on_pcm(pcm, event) if codec_eos: break if codec_done_time is None: codec_done_time = time.monotonic() if pipeline is not None: pipeline.mark_codec_finished(codec_eos) # Finish Thinker if Talker stopped too early; record this outcome, don't hide it. text_finished_before_codec = answer.done with torch.autocast('cuda', dtype=torch.bfloat16): while not answer.done: answer.pop() if pipeline is not None: pipeline.finish() elif decoder is not None: with torch.autocast('cuda', enabled=False): for pcm,event in decoder.finish(): event['codec_eos_observed'] = codec_eos on_pcm(pcm,event) text_result = answer.result() if trace is not None: trace.mark_once('native_thinker_end') trace.set('generated_text_tokens', len(text_result.get('ids', []))) trace.set('generated_codec_count', len(new_ids)) trace.set('first_chunk_codec_count', int(first_chunk_codes or (geometry.chunk_codes if geometry else SpeechGeometry().chunk_codes))) return dict(native_speech=decoder is not None, **text_result, speaker=speaker, codec_ids=new_ids, codec_tokens=len(new_ids), talker_eos=codec_eos, first_codec_monotonic=first_codec_time, codec_complete_monotonic=codec_done_time, decode_speech_seconds=time.monotonic()-before, text_finished_before_codec=text_finished_before_codec, solver=solver, num_steps=num_steps, code_only=code_only, decoder_pipeline='parallel' if pipeline is not None else 'serial', decoder_device=str(decoder_device), codec_sync_interval=int(codec_sync_interval), preallocate_sequence=bool(preallocate_sequence), fast_talker_step=bool(fast_talker_step), fast_sampling=bool(fast_sampling), fused_flash_kv=bool(fused_flash_kv), decoder_max_pending=0 if pipeline is None else pipeline.max_pending_observed, decoder_backpressure_seconds=0. if pipeline is None else pipeline.backpressure_seconds, audio_chunks=[] if decoder is None else decoder.events, native_streaming_scope='incremental text + native codec + native PCM; concurrent body/input requires separate worker', samples=0 if decoder is None else decoder.emitted_samples) except BaseException as error: if trace is not None: trace.set('cancelled', True) trace.set('cancel_reason', type(error).__name__) trace.mark_once('native_thinker_end') raise finally: if pipeline is not None: pipeline.close()