Nafiul Huda commited on
Commit
05d9aea
Β·
verified Β·
1 Parent(s): e57a2b8

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +88 -41
app.py CHANGED
@@ -1,13 +1,14 @@
1
  import asyncio
2
  import re
3
  import os
 
 
4
 
5
  import torch
6
  import gradio as gr
7
  import edge_tts
8
  import spaces
9
  from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
10
- from threading import Thread
11
 
12
  # ──────────────────────────────────────────────────────────────────────────────
13
  # CONFIG
@@ -36,23 +37,40 @@ print(f"[model] loaded on: {next(model.parameters()).device}")
36
 
37
 
38
  # ──────────────────────────────────────────────────────────────────────────────
39
- # TTS β€” persistent event loop
40
  # ──────────────────────────────────────────────────────────────────────────────
41
- _tts_loop = None
42
-
43
- def _get_or_create_loop():
44
- global _tts_loop
45
- if _tts_loop is None or _tts_loop.is_closed():
46
- _tts_loop = asyncio.new_event_loop()
47
- return _tts_loop
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
48
 
49
- async def _tts_async(text: str, path: str):
50
- await edge_tts.Communicate(text, TTS_VOICE).save(path)
 
51
 
52
  def text_to_speech(text: str) -> str:
53
- loop = _get_or_create_loop()
54
- loop.run_until_complete(_tts_async(text, AUDIO_OUT))
55
- return AUDIO_OUT
 
56
 
57
 
58
  # ──────────────────────────────────────────────────────────────────────────────
@@ -68,7 +86,10 @@ def chat_and_speak(
68
  enable_thinking: bool,
69
  enable_tts: bool,
70
  ):
71
- # Build messages β€” Gradio 5 uses [{"role": ..., "content": ...}, ...]
 
 
 
72
  messages = [{"role": "system", "content": system_prompt}]
73
  messages.extend(history)
74
  messages.append({"role": "user", "content": user_message})
@@ -83,7 +104,7 @@ def chat_and_speak(
83
  return_tensors = "pt",
84
  ).to(model.device)
85
 
86
- # Streaming
87
  streamer = TextIteratorStreamer(
88
  tokenizer,
89
  skip_prompt = True,
@@ -102,7 +123,7 @@ def chat_and_speak(
102
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
103
  thread.start()
104
 
105
- # Stream
106
  partial_reply = ""
107
  current_history = history + [
108
  {"role": "user", "content": user_message},
@@ -116,7 +137,7 @@ def chat_and_speak(
116
 
117
  thread.join()
118
 
119
- # Strip <think> blocks
120
  clean_reply = re.sub(
121
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
122
  ).strip()
@@ -145,9 +166,9 @@ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
145
 
146
  gr.Markdown(
147
  r"""
148
- # πŸ€– MiniCPM5-1B Chat
149
  **Model:** `openbmb/MiniCPM5-1B` (FP16 Β· Transformers) &nbsp;|&nbsp;
150
- **TTS:** `zh-CN-XiaoyiNeural`
151
  """
152
  )
153
 
@@ -156,62 +177,88 @@ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
156
  chatbot = gr.Chatbot(
157
  elem_id="chatbot",
158
  label="Conversation",
 
159
  )
160
  with gr.Row():
161
  user_input = gr.Textbox(
162
- placeholder="Type your message …",
163
  label="Your message",
164
- lines=2, scale=5,
 
165
  )
166
  send_btn = gr.Button("Send πŸš€", variant="primary", scale=1)
167
- clear_btn = gr.Button("πŸ—‘οΈ Clear", variant="secondary")
168
  audio_out = gr.Audio(
169
- label="πŸ”Š TTS β€” XiaoyiNeural",
170
- type="filepath", autoplay=True,
 
171
  )
172
 
173
  with gr.Column(scale=1):
174
  gr.Markdown("### βš™οΈ Settings")
 
175
  system_prompt = gr.Textbox(
176
- value="You are a helpful, concise assistant.",
177
- label="System Prompt",
178
  lines=3,
 
179
  )
 
180
  max_new_tokens = gr.Slider(
181
- 64, 4096, value=512, step=64, label="Max new tokens"
 
 
 
 
182
  )
 
183
  temperature = gr.Slider(
184
- 0.1, 2.0, value=0.7, step=0.05, label="Temperature"
 
 
 
 
 
185
  )
 
186
  enable_thinking = gr.Checkbox(
187
  value=False,
188
- label="🧠 Enable Thinking (<think> mode)",
 
189
  )
 
190
  enable_tts = gr.Checkbox(
191
- value=True, label="πŸ”Š Enable TTS (XiaoyiNeural)"
 
 
192
  )
 
193
  gr.Markdown(
194
  r"""
195
  ---
196
- | | |
197
- |---|---|
198
- | Model | MiniCPM5-1B FP16 |
199
- | Size | 2.17 GB |
200
- | Arch | LlamaForCausalLM |
201
- | Backend | Transformers 5.6+ |
202
- | Voice | XiaoyiNeural |
203
  """
204
  )
205
 
206
  state = gr.State([])
207
 
 
208
  for trigger in [send_btn.click, user_input.submit]:
209
  trigger(
210
  fn = chat_and_speak,
211
  inputs = [
212
- user_input, state, system_prompt,
213
- max_new_tokens, temperature,
214
- enable_thinking, enable_tts,
 
 
 
 
215
  ],
216
  outputs = [chatbot, state, audio_out],
217
  ).then(lambda: "", outputs=user_input)
 
1
  import asyncio
2
  import re
3
  import os
4
+ from threading import Thread
5
+ from queue import Queue
6
 
7
  import torch
8
  import gradio as gr
9
  import edge_tts
10
  import spaces
11
  from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 
12
 
13
  # ──────────────────────────────────────────────────────────────────────────────
14
  # CONFIG
 
37
 
38
 
39
  # ──────────────────────────────────────────────────────────────────────────────
40
+ # TTS β€” background thread with persistent event loop (no asyncio warnings)
41
  # ──────────────────────────────────────────────────────────────────────────────
42
+ _tts_queue = Queue()
43
+ _tts_result_queue = Queue()
44
+
45
+ def _tts_worker():
46
+ """Background thread that runs its own event loop for TTS."""
47
+ loop = asyncio.new_event_loop()
48
+ asyncio.set_event_loop(loop)
49
+
50
+ while True:
51
+ task = _tts_queue.get()
52
+ if task is None: # poison pill
53
+ break
54
+
55
+ text, path = task
56
+ try:
57
+ loop.run_until_complete(
58
+ edge_tts.Communicate(text, TTS_VOICE).save(path)
59
+ )
60
+ _tts_result_queue.put(path)
61
+ except Exception as e:
62
+ _tts_result_queue.put(None)
63
+ print(f"[TTS] error: {e}")
64
 
65
+ # Start TTS worker thread
66
+ _tts_thread = Thread(target=_tts_worker, daemon=True)
67
+ _tts_thread.start()
68
 
69
  def text_to_speech(text: str) -> str:
70
+ """Thread-safe TTS β€” no asyncio warnings."""
71
+ _tts_queue.put((text, AUDIO_OUT))
72
+ result = _tts_result_queue.get()
73
+ return result if result else None
74
 
75
 
76
  # ──────────────────────────────────────────────────────────────────────────────
 
86
  enable_thinking: bool,
87
  enable_tts: bool,
88
  ):
89
+ if not user_message.strip():
90
+ return history, history, None
91
+
92
+ # Build messages for model β€” include system prompt
93
  messages = [{"role": "system", "content": system_prompt}]
94
  messages.extend(history)
95
  messages.append({"role": "user", "content": user_message})
 
104
  return_tensors = "pt",
105
  ).to(model.device)
106
 
107
+ # Streaming setup
108
  streamer = TextIteratorStreamer(
109
  tokenizer,
110
  skip_prompt = True,
 
123
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
124
  thread.start()
125
 
126
+ # Stream tokens
127
  partial_reply = ""
128
  current_history = history + [
129
  {"role": "user", "content": user_message},
 
137
 
138
  thread.join()
139
 
140
+ # Clean <think> blocks for display/TTS
141
  clean_reply = re.sub(
142
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
143
  ).strip()
 
166
 
167
  gr.Markdown(
168
  r"""
169
+ # πŸ€– MiniCPM5-1B Chat with TTS
170
  **Model:** `openbmb/MiniCPM5-1B` (FP16 Β· Transformers) &nbsp;|&nbsp;
171
+ **TTS Voice:** `zh-CN-XiaoyiNeural` (Xiaoyi)
172
  """
173
  )
174
 
 
177
  chatbot = gr.Chatbot(
178
  elem_id="chatbot",
179
  label="Conversation",
180
+ height=500,
181
  )
182
  with gr.Row():
183
  user_input = gr.Textbox(
184
+ placeholder="Type your message here…",
185
  label="Your message",
186
+ lines=2,
187
+ scale=5,
188
  )
189
  send_btn = gr.Button("Send πŸš€", variant="primary", scale=1)
190
+ clear_btn = gr.Button("πŸ—‘οΈ Clear conversation", variant="secondary")
191
  audio_out = gr.Audio(
192
+ label="πŸ”Š TTS Output (XiaoyiNeural)",
193
+ type="filepath",
194
+ autoplay=True,
195
  )
196
 
197
  with gr.Column(scale=1):
198
  gr.Markdown("### βš™οΈ Settings")
199
+
200
  system_prompt = gr.Textbox(
201
+ value="You are a helpful, friendly, and concise AI assistant.",
202
+ label="πŸ’¬ System Prompt",
203
  lines=3,
204
+ info="Sets the assistant's personality and behavior",
205
  )
206
+
207
  max_new_tokens = gr.Slider(
208
+ minimum=64,
209
+ maximum=4096,
210
+ value=512,
211
+ step=64,
212
+ label="πŸ“ Max new tokens",
213
  )
214
+
215
  temperature = gr.Slider(
216
+ minimum=0.1,
217
+ maximum=2.0,
218
+ value=0.7,
219
+ step=0.05,
220
+ label="🌑️ Temperature",
221
+ info="Higher = more creative, lower = more focused",
222
  )
223
+
224
  enable_thinking = gr.Checkbox(
225
  value=False,
226
+ label="🧠 Enable Chain-of-Thought (<think> mode)",
227
+ info="Model shows its reasoning process",
228
  )
229
+
230
  enable_tts = gr.Checkbox(
231
+ value=True,
232
+ label="πŸ”Š Enable TTS (XiaoyiNeural)",
233
+ info="Speak responses aloud",
234
  )
235
+
236
  gr.Markdown(
237
  r"""
238
  ---
239
+ **Model Info:**
240
+ - **Name:** MiniCPM5-1B
241
+ - **Size:** 2.17 GB (FP16)
242
+ - **Context:** 8K tokens
243
+ - **Backend:** Transformers + PyTorch
244
+ - **Voice:** zh-CN-XiaoyiNeural
 
245
  """
246
  )
247
 
248
  state = gr.State([])
249
 
250
+ # Wire events
251
  for trigger in [send_btn.click, user_input.submit]:
252
  trigger(
253
  fn = chat_and_speak,
254
  inputs = [
255
+ user_input,
256
+ state,
257
+ system_prompt,
258
+ max_new_tokens,
259
+ temperature,
260
+ enable_thinking,
261
+ enable_tts,
262
  ],
263
  outputs = [chatbot, state, audio_out],
264
  ).then(lambda: "", outputs=user_input)