Nafiul Huda commited on
Commit
a6ee577
Β·
verified Β·
1 Parent(s): 05d9aea

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +45 -59
app.py CHANGED
@@ -2,12 +2,10 @@ import asyncio
2
  import re
3
  import os
4
  from threading import Thread
5
- from queue import Queue
6
 
7
  import torch
8
  import gradio as gr
9
  import edge_tts
10
- import spaces
11
  from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
12
 
13
  # ──────────────────────────────────────────────────────────────────────────────
@@ -17,66 +15,55 @@ MODEL_ID = "openbmb/MiniCPM5-1B"
17
  TTS_VOICE = "zh-CN-XiaoyiNeural"
18
  AUDIO_OUT = "/tmp/tts_output.mp3"
19
 
 
 
 
 
 
20
  # ──────────────────────────────────────────────────────────────────────────────
21
- # MODEL LOAD
22
  # ──────────────────────────────────────────────────────────────────────────────
23
  print("=" * 60)
24
- print(" Loading openbmb/MiniCPM5-1B (FP16/BF16 Transformers)")
25
  print("=" * 60)
26
 
27
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
28
 
29
  model = AutoModelForCausalLM.from_pretrained(
30
  MODEL_ID,
31
- torch_dtype = "auto",
32
- device_map = "auto",
33
  trust_remote_code = True,
 
34
  )
35
  model.eval()
36
  print(f"[model] loaded on: {next(model.parameters()).device}")
 
37
 
38
 
39
  # ──────────────────────────────────────────────────────────────────────────────
40
- # TTS β€” background thread with persistent event loop (no asyncio warnings)
41
  # ──────────────────────────────────────────────────────────────────────────────
42
- _tts_queue = Queue()
43
- _tts_result_queue = Queue()
44
 
45
- def _tts_worker():
46
- """Background thread that runs its own event loop for TTS."""
47
- loop = asyncio.new_event_loop()
48
- asyncio.set_event_loop(loop)
49
-
50
- while True:
51
- task = _tts_queue.get()
52
- if task is None: # poison pill
53
- break
54
-
55
- text, path = task
56
- try:
57
- loop.run_until_complete(
58
- edge_tts.Communicate(text, TTS_VOICE).save(path)
59
- )
60
- _tts_result_queue.put(path)
61
- except Exception as e:
62
- _tts_result_queue.put(None)
63
- print(f"[TTS] error: {e}")
64
 
65
- # Start TTS worker thread
66
- _tts_thread = Thread(target=_tts_worker, daemon=True)
67
- _tts_thread.start()
68
 
69
  def text_to_speech(text: str) -> str:
70
- """Thread-safe TTS β€” no asyncio warnings."""
71
- _tts_queue.put((text, AUDIO_OUT))
72
- result = _tts_result_queue.get()
73
- return result if result else None
74
 
75
 
76
  # ──────────────────────────────────────────────────────────────────────────────
77
- # INFERENCE
78
  # ──────────────────────────────────────────────────────────────────────────────
79
- @spaces.GPU(duration=120)
80
  def chat_and_speak(
81
  user_message: str,
82
  history: list,
@@ -89,7 +76,7 @@ def chat_and_speak(
89
  if not user_message.strip():
90
  return history, history, None
91
 
92
- # Build messages for model β€” include system prompt
93
  messages = [{"role": "system", "content": system_prompt}]
94
  messages.extend(history)
95
  messages.append({"role": "user", "content": user_message})
@@ -102,9 +89,9 @@ def chat_and_speak(
102
  enable_thinking = enable_thinking,
103
  return_dict = True,
104
  return_tensors = "pt",
105
- ).to(model.device)
106
 
107
- # Streaming setup
108
  streamer = TextIteratorStreamer(
109
  tokenizer,
110
  skip_prompt = True,
@@ -118,6 +105,7 @@ def chat_and_speak(
118
  do_sample = temperature > 0,
119
  streamer = streamer,
120
  pad_token_id = tokenizer.eos_token_id,
 
121
  )
122
 
123
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
@@ -137,7 +125,7 @@ def chat_and_speak(
137
 
138
  thread.join()
139
 
140
- # Clean <think> blocks for display/TTS
141
  clean_reply = re.sub(
142
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
143
  ).strip()
@@ -166,9 +154,9 @@ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
166
 
167
  gr.Markdown(
168
  r"""
169
- # πŸ€– MiniCPM5-1B Chat with TTS
170
- **Model:** `openbmb/MiniCPM5-1B` (FP16 Β· Transformers) &nbsp;|&nbsp;
171
- **TTS Voice:** `zh-CN-XiaoyiNeural` (Xiaoyi)
172
  """
173
  )
174
 
@@ -201,12 +189,12 @@ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
201
  value="You are a helpful, friendly, and concise AI assistant.",
202
  label="πŸ’¬ System Prompt",
203
  lines=3,
204
- info="Sets the assistant's personality and behavior",
205
  )
206
 
207
  max_new_tokens = gr.Slider(
208
  minimum=64,
209
- maximum=4096,
210
  value=512,
211
  step=64,
212
  label="πŸ“ Max new tokens",
@@ -218,36 +206,34 @@ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
218
  value=0.7,
219
  step=0.05,
220
  label="🌑️ Temperature",
221
- info="Higher = more creative, lower = more focused",
222
  )
223
 
224
  enable_thinking = gr.Checkbox(
225
  value=False,
226
- label="🧠 Enable Chain-of-Thought (<think> mode)",
227
- info="Model shows its reasoning process",
228
  )
229
 
230
  enable_tts = gr.Checkbox(
231
  value=True,
232
- label="πŸ”Š Enable TTS (XiaoyiNeural)",
233
- info="Speak responses aloud",
234
  )
235
 
236
  gr.Markdown(
237
- r"""
238
  ---
239
- **Model Info:**
240
- - **Name:** MiniCPM5-1B
241
- - **Size:** 2.17 GB (FP16)
242
- - **Context:** 8K tokens
243
- - **Backend:** Transformers + PyTorch
244
- - **Voice:** zh-CN-XiaoyiNeural
245
  """
246
  )
247
 
248
  state = gr.State([])
249
 
250
- # Wire events
251
  for trigger in [send_btn.click, user_input.submit]:
252
  trigger(
253
  fn = chat_and_speak,
 
2
  import re
3
  import os
4
  from threading import Thread
 
5
 
6
  import torch
7
  import gradio as gr
8
  import edge_tts
 
9
  from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
10
 
11
  # ──────────────────────────────────────────────────────────────────────────────
 
15
  TTS_VOICE = "zh-CN-XiaoyiNeural"
16
  AUDIO_OUT = "/tmp/tts_output.mp3"
17
 
18
+ # Use all available CPU cores
19
+ CPU_THREADS = os.cpu_count() or 4
20
+ torch.set_num_threads(CPU_THREADS)
21
+ os.environ["OMP_NUM_THREADS"] = str(CPU_THREADS)
22
+
23
  # ──────────────────────────────────────────────────────────────────────────────
24
+ # MODEL LOAD β€” CPU ONLY
25
  # ──────────────────────────────────────────────────────────────────────────────
26
  print("=" * 60)
27
+ print(f" Loading MiniCPM5-1B on CPU ({CPU_THREADS} threads)")
28
  print("=" * 60)
29
 
30
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
31
 
32
  model = AutoModelForCausalLM.from_pretrained(
33
  MODEL_ID,
34
+ torch_dtype = torch.float32, # CPU doesn't support bf16 efficiently
35
+ device_map = "cpu", # force CPU
36
  trust_remote_code = True,
37
+ low_cpu_mem_usage = True,
38
  )
39
  model.eval()
40
  print(f"[model] loaded on: {next(model.parameters()).device}")
41
+ print(f"[model] using {CPU_THREADS} CPU threads")
42
 
43
 
44
  # ──────────────────────────────────────────────────────────────────────────────
45
+ # TTS β€” persistent event loop (the version that worked)
46
  # ──────────────────────────────────────────────────────────────────────────────
47
+ _tts_loop = None
 
48
 
49
+ def _get_or_create_loop():
50
+ global _tts_loop
51
+ if _tts_loop is None or _tts_loop.is_closed():
52
+ _tts_loop = asyncio.new_event_loop()
53
+ return _tts_loop
 
 
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
+ async def _tts_async(text: str, path: str):
56
+ await edge_tts.Communicate(text, TTS_VOICE).save(path)
 
57
 
58
  def text_to_speech(text: str) -> str:
59
+ loop = _get_or_create_loop()
60
+ loop.run_until_complete(_tts_async(text, AUDIO_OUT))
61
+ return AUDIO_OUT
 
62
 
63
 
64
  # ──────────────────────────────────────────────────────────────────────────────
65
+ # INFERENCE β€” NO GPU, CPU ONLY
66
  # ──────────────────────────────────────────────────────────────────────────────
 
67
  def chat_and_speak(
68
  user_message: str,
69
  history: list,
 
76
  if not user_message.strip():
77
  return history, history, None
78
 
79
+ # Build messages
80
  messages = [{"role": "system", "content": system_prompt}]
81
  messages.extend(history)
82
  messages.append({"role": "user", "content": user_message})
 
89
  enable_thinking = enable_thinking,
90
  return_dict = True,
91
  return_tensors = "pt",
92
+ )
93
 
94
+ # Streaming
95
  streamer = TextIteratorStreamer(
96
  tokenizer,
97
  skip_prompt = True,
 
105
  do_sample = temperature > 0,
106
  streamer = streamer,
107
  pad_token_id = tokenizer.eos_token_id,
108
+ num_threads = CPU_THREADS, # ensure all threads are used
109
  )
110
 
111
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
 
125
 
126
  thread.join()
127
 
128
+ # Strip <think> blocks
129
  clean_reply = re.sub(
130
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
131
  ).strip()
 
154
 
155
  gr.Markdown(
156
  r"""
157
+ # πŸ€– MiniCPM5-1B Chat (CPU-only)
158
+ **Model:** `openbmb/MiniCPM5-1B` (FP32 Β· CPU) &nbsp;|&nbsp;
159
+ **TTS Voice:** `zh-CN-XiaoyiNeural`
160
  """
161
  )
162
 
 
189
  value="You are a helpful, friendly, and concise AI assistant.",
190
  label="πŸ’¬ System Prompt",
191
  lines=3,
192
+ info="Sets the assistant's personality",
193
  )
194
 
195
  max_new_tokens = gr.Slider(
196
  minimum=64,
197
+ maximum=2048,
198
  value=512,
199
  step=64,
200
  label="πŸ“ Max new tokens",
 
206
  value=0.7,
207
  step=0.05,
208
  label="🌑️ Temperature",
209
+ info="Higher = more creative",
210
  )
211
 
212
  enable_thinking = gr.Checkbox(
213
  value=False,
214
+ label="🧠 Enable Chain-of-Thought",
215
+ info="Shows reasoning in <think> tags",
216
  )
217
 
218
  enable_tts = gr.Checkbox(
219
  value=True,
220
+ label="πŸ”Š Enable TTS",
221
+ info="XiaoyiNeural voice",
222
  )
223
 
224
  gr.Markdown(
225
+ f"""
226
  ---
227
+ **Runtime:**
228
+ - Device: CPU
229
+ - Threads: {CPU_THREADS}
230
+ - Model: FP32 (2.17 GB)
231
+ - Voice: zh-CN-XiaoyiNeural
 
232
  """
233
  )
234
 
235
  state = gr.State([])
236
 
 
237
  for trigger in [send_btn.click, user_input.submit]:
238
  trigger(
239
  fn = chat_and_speak,