EYEDOL commited on
Commit
1dd4e6c
·
verified ·
1 Parent(s): e22fdd2

Delete pipeline.py

Browse files
Files changed (1) hide show
  1. pipeline.py +0 -231
pipeline.py DELETED
@@ -1,231 +0,0 @@
1
- """
2
- pipeline.py
3
- -----------
4
- Core streaming overlap pipeline.
5
-
6
- Flow
7
- ----
8
- Conversation history
9
- │
10
- ▼
11
- SmolLM2 (thread) ──tokens──► SentenceBuffer
12
- │
13
- sentence complete?
14
- │ yes
15
- ▼
16
- MMS-TTS (executor) ← runs while LLM
17
- │ keeps generating!
18
- ▼
19
- ws.send_bytes(pcm)
20
- │
21
- ◄─────────┘ repeat
22
-
23
- The key insight: TTS synthesis for sentence N starts as soon as sentence N is
24
- complete — the LLM does NOT wait. By the time the browser finishes playing
25
- sentence N, sentence N+1 is already synthesised and queued, giving near-zero
26
- inter-sentence gaps and a much lower perceived total latency.
27
- """
28
-
29
- from __future__ import annotations
30
-
31
- import asyncio
32
- import logging
33
- import time
34
- from threading import Thread
35
- from typing import AsyncGenerator
36
-
37
- from fastapi import WebSocket
38
- from transformers import TextIteratorStreamer # type: ignore
39
-
40
- from models import MAX_NEW_TOKENS, ModelManager
41
- from sentence_buffer import SentenceBuffer
42
-
43
- logger = logging.getLogger(__name__)
44
-
45
- # Maximum conversation turns kept in context (system + N user/assistant pairs)
46
- MAX_HISTORY_TURNS = 6
47
- SYSTEM_PROMPT = (
48
- "You are a helpful, friendly voice assistant. "
49
- "Keep every reply concise — two or three short sentences at most. "
50
- "Speak naturally; avoid bullet points, markdown, or lists."
51
- )
52
-
53
-
54
- # --------------------------------------------------------------------------- #
55
- # Async LLM token stream #
56
- # --------------------------------------------------------------------------- #
57
- async def _llm_token_stream(
58
- models: ModelManager,
59
- messages: list[dict],
60
- loop: asyncio.AbstractEventLoop,
61
- ) -> AsyncGenerator[str, None]:
62
- """
63
- Async generator that yields text tokens from SmolLM2 as they are produced.
64
-
65
- Architecture
66
- ~~~~~~~~~~~~
67
- model.generate() runs in gen_thread (CPU/GPU bound).
68
- TextIteratorStreamer bridges the sync world → async Queue via
69
- run_coroutine_threadsafe so the FastAPI event loop stays unblocked.
70
- """
71
- prompt = models.build_prompt(messages)
72
- enc = models.tokenize(prompt)
73
-
74
- streamer = TextIteratorStreamer(
75
- models.llm_tokenizer,
76
- skip_prompt=True,
77
- skip_special_tokens=True,
78
- timeout=60.0,
79
- )
80
-
81
- gen_kwargs = dict(
82
- input_ids=enc["input_ids"],
83
- attention_mask=enc.get("attention_mask"),
84
- streamer=streamer,
85
- max_new_tokens=MAX_NEW_TOKENS,
86
- do_sample=True,
87
- temperature=0.7,
88
- repetition_penalty=1.1,
89
- pad_token_id=models.llm_tokenizer.eos_token_id,
90
- )
91
-
92
- token_q: asyncio.Queue[str | None] = asyncio.Queue()
93
-
94
- # ── thread: start generation + drain streamer into async queue ──────── #
95
- def _generate_and_feed():
96
- gen_thread = Thread(
97
- target=models.llm.generate, kwargs=gen_kwargs, daemon=True
98
- )
99
- gen_thread.start()
100
- try:
101
- for tok in streamer: # blocks until each token is ready
102
- asyncio.run_coroutine_threadsafe(token_q.put(tok), loop)
103
- except Exception as exc:
104
- asyncio.run_coroutine_threadsafe(token_q.put(None), loop)
105
- logger.error("LLM stream error: %s", exc)
106
- return
107
- asyncio.run_coroutine_threadsafe(token_q.put(None), loop) # sentinel
108
-
109
- feed_thread = Thread(target=_generate_and_feed, daemon=True)
110
- feed_thread.start()
111
-
112
- # ── yield tokens from the queue ─────────────────────────────────────── #
113
- while True:
114
- tok = await token_q.get()
115
- if tok is None:
116
- break
117
- yield tok
118
-
119
-
120
- # --------------------------------------------------------------------------- #
121
- # StreamingPipeline #
122
- # --------------------------------------------------------------------------- #
123
- class StreamingPipeline:
124
- """
125
- Orchestrates the full LLM → SentenceBuffer → TTS → WebSocket pipeline
126
- with streaming overlap.
127
- """
128
-
129
- def __init__(self, models: ModelManager):
130
- self.models = models
131
- self._executor = None # Use default loop executor (ThreadPoolExecutor)
132
-
133
- async def process(
134
- self,
135
- conversation: list[dict],
136
- ws: WebSocket,
137
- loop: asyncio.AbstractEventLoop,
138
- ) -> str:
139
- """
140
- Stream an LLM response over `ws` as audio chunks.
141
-
142
- Returns the full text of the assistant turn (for appending to history).
143
-
144
- Parameters
145
- ----------
146
- conversation : list of {role, content} dicts including the new user turn.
147
- ws : active WebSocket connection to send PCM bytes to.
148
- loop : running event loop (passed for thread-safe queue ops).
149
- """
150
- # Trim history to avoid context overflow
151
- conversation = _trim_history(conversation)
152
-
153
- sentence_buf = SentenceBuffer(min_length=8)
154
- full_text = ""
155
- t_start = time.perf_counter()
156
- ttft_logged = False
157
-
158
- async for token in _llm_token_stream(self.models, conversation, loop):
159
- if not ttft_logged:
160
- ttft_ms = (time.perf_counter() - t_start) * 1000
161
- logger.info("LLM TTFT: %.0f ms", ttft_ms)
162
- ttft_logged = True
163
-
164
- full_text += token
165
- ready = sentence_buf.add(token)
166
-
167
- for sentence in ready:
168
- await self._synth_and_send(sentence, ws)
169
-
170
- # Flush any partial sentence left in buffer
171
- tail = sentence_buf.flush()
172
- if tail:
173
- await self._synth_and_send(tail, ws)
174
-
175
- total_ms = (time.perf_counter() - t_start) * 1000
176
- logger.info(
177
- "Pipeline done in %.0f ms | response: %d chars", total_ms, len(full_text)
178
- )
179
- return full_text.strip()
180
-
181
- # ------------------------------------------------------------------ #
182
- async def _synth_and_send(self, sentence: str, ws: WebSocket):
183
- """
184
- Synthesise one sentence and stream PCM bytes over the WebSocket.
185
-
186
- TTS runs in the default thread-pool executor so the event loop
187
- remains free to handle other coroutines (e.g. interruption frames
188
- arriving from the client) during synthesis.
189
- """
190
- sentence = sentence.strip()
191
- if not sentence:
192
- return
193
-
194
- logger.debug("TTS ← '%s'", sentence[:80])
195
- t0 = time.perf_counter()
196
-
197
- pcm: bytes | None = await asyncio.get_event_loop().run_in_executor(
198
- None, self.models.synthesize, sentence
199
- )
200
-
201
- if not pcm:
202
- return
203
-
204
- tts_ms = (time.perf_counter() - t0) * 1000
205
- logger.info("TTS (%.0f ms): %d bytes for '%s…'", tts_ms, len(pcm), sentence[:40])
206
-
207
- # Send in 4 kB chunks to give the client a smooth receive stream
208
- CHUNK = 4096
209
- for i in range(0, len(pcm), CHUNK):
210
- await ws.send_bytes(pcm[i : i + CHUNK])
211
-
212
-
213
- # --------------------------------------------------------------------------- #
214
- # Helpers #
215
- # --------------------------------------------------------------------------- #
216
- def _trim_history(messages: list[dict]) -> list[dict]:
217
- """Keep system message + last MAX_HISTORY_TURNS user/assistant pairs."""
218
- system = [m for m in messages if m["role"] == "system"]
219
- turns = [m for m in messages if m["role"] != "system"]
220
-
221
- # Each "turn" = 1 user + 1 assistant message = 2 items
222
- max_items = MAX_HISTORY_TURNS * 2
223
- if len(turns) > max_items:
224
- turns = turns[-max_items:]
225
-
226
- return system + turns
227
-
228
-
229
- def build_initial_conversation() -> list[dict]:
230
- """Start a fresh conversation with only the system prompt."""
231
- return [{"role": "system", "content": SYSTEM_PROMPT}]