Nafiul Huda commited on
Commit
709b31d
Β·
verified Β·
1 Parent(s): b25c712

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +49 -43
app.py CHANGED
@@ -17,9 +17,7 @@ TTS_VOICE = "zh-CN-XiaoyiNeural"
17
  AUDIO_OUT = "/tmp/tts_output.mp3"
18
 
19
  # ──────────────────────────────────────────────────────────────────────────────
20
- # MODEL LOAD β€” runs once at Space boot, outside @spaces.GPU
21
- # torch_dtype="auto" β†’ bfloat16 on ZeroGPU A100, fp16 elsewhere
22
- # device_map="auto" β†’ moves everything to CUDA when available
23
  # ──────────────────────────────────────────────────────────────────────────────
24
  print("=" * 60)
25
  print(" Loading openbmb/MiniCPM5-1B (FP16/BF16 Transformers)")
@@ -29,8 +27,8 @@ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
29
 
30
  model = AutoModelForCausalLM.from_pretrained(
31
  MODEL_ID,
32
- torch_dtype = "auto", # bf16 on Ampere+, fp16 elsewhere
33
- device_map = "auto", # fill GPU first, overflow to CPU
34
  trust_remote_code = True,
35
  )
36
  model.eval()
@@ -38,83 +36,94 @@ print(f"[model] loaded on: {next(model.parameters()).device}")
38
 
39
 
40
  # ──────────────────────────────────────────────────────────────────────────────
41
- # TTS β€” XiaoyiNeural (locked)
42
  # ──────────────────────────────────────────────────────────────────────────────
 
 
 
 
 
 
 
 
43
  async def _tts_async(text: str, path: str):
44
  await edge_tts.Communicate(text, TTS_VOICE).save(path)
45
 
46
  def text_to_speech(text: str) -> str:
47
- asyncio.run(_tts_async(text, AUDIO_OUT))
 
48
  return AUDIO_OUT
49
 
50
 
51
  # ──────────────────────────────────────────────────────────────────────────────
52
- # INFERENCE (ZeroGPU β€” GPU is live for the duration of this call)
53
  # ──────────────────────────────────────────────────────────────────────────────
54
  @spaces.GPU(duration=120)
55
  def chat_and_speak(
56
  user_message: str,
57
- history: list, # [[human, bot], ...]
58
  system_prompt: str,
59
  max_new_tokens: int,
60
  temperature: float,
61
  enable_thinking: bool,
62
  enable_tts: bool,
63
  ):
64
- # Build messages list
65
  messages = [{"role": "system", "content": system_prompt}]
66
- for human, bot in history:
67
- messages.append({"role": "user", "content": human})
68
- messages.append({"role": "assistant", "content": bot})
69
  messages.append({"role": "user", "content": user_message})
70
 
71
- # Tokenise with the official MiniCPM5 chat template
72
  inputs = tokenizer.apply_chat_template(
73
  messages,
74
- tokenize = True,
75
- add_generation_prompt= True,
76
- enable_thinking = enable_thinking, # True β†’ <think>…</think> reasoning
77
- return_dict = True,
78
- return_tensors = "pt",
79
  ).to(model.device)
80
 
81
- # Streaming via TextIteratorStreamer
82
  streamer = TextIteratorStreamer(
83
  tokenizer,
84
- skip_prompt = True,
85
- skip_special_tokens = True,
86
  )
87
 
88
  gen_kwargs = dict(
89
  **inputs,
90
- max_new_tokens = max_new_tokens,
91
- temperature = temperature,
92
- do_sample = temperature > 0,
93
- streamer = streamer,
94
- pad_token_id = tokenizer.eos_token_id,
95
  )
96
 
97
- # Run generation in a background thread so we can stream tokens
98
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
99
  thread.start()
100
 
101
- # Collect streamed tokens and yield partial updates to Gradio
102
  partial_reply = ""
103
- current_history = history + [[user_message, ""]]
 
 
 
104
 
105
  for new_token in streamer:
106
  partial_reply += new_token
107
- current_history[-1][1] = partial_reply
108
- yield current_history, current_history, None # stream to chatbot
109
 
110
  thread.join()
111
 
112
- # Strip <think>…</think> blocks for display & TTS
113
  clean_reply = re.sub(
114
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
115
  ).strip()
116
 
117
- current_history[-1][1] = clean_reply
118
 
119
  # TTS
120
  audio_path = None
@@ -131,9 +140,7 @@ def clear_all():
131
  # ──────────────────────────────────────────────────────────────────────────────
132
  # GRADIO UI
133
  # ──────────────────────────────────────────────────────────────────────────────
134
- with gr.Blocks(
135
- title="MiniCPM5-1B Chat",
136
- ) as demo:
137
 
138
  demo.theme = gr.themes.Soft()
139
  demo.css = "#chatbot{height:500px} footer{display:none!important}"
@@ -147,9 +154,12 @@ with gr.Blocks(
147
  )
148
 
149
  with gr.Row():
150
- # ── Left: chat ──────────────────────────────────────────────────────
151
  with gr.Column(scale=3):
152
- chatbot = gr.Chatbot(elem_id="chatbot", label="Conversation")
 
 
 
 
153
  with gr.Row():
154
  user_input = gr.Textbox(
155
  placeholder="Type your message …",
@@ -163,7 +173,6 @@ with gr.Blocks(
163
  type="filepath", autoplay=True,
164
  )
165
 
166
- # ── Right: settings ─────────────────────────────────────────────────
167
  with gr.Column(scale=1):
168
  gr.Markdown("### βš™οΈ Settings")
169
  system_prompt = gr.Textbox(
@@ -199,7 +208,6 @@ with gr.Blocks(
199
 
200
  state = gr.State([])
201
 
202
- # ── Wire send button & Enter key ────────────────────────────────────────
203
  for trigger in [send_btn.click, user_input.submit]:
204
  trigger(
205
  fn = chat_and_speak,
@@ -218,6 +226,4 @@ if __name__ == "__main__":
218
  demo.queue(max_size=5).launch(
219
  server_name="0.0.0.0",
220
  server_port=7860,
221
- theme=gr.themes.Soft(),
222
- css="#chatbot{height:500px} footer{display:none!important}",
223
  )
 
17
  AUDIO_OUT = "/tmp/tts_output.mp3"
18
 
19
  # ──────────────────────────────────────────────────────────────────────────────
20
+ # MODEL LOAD
 
 
21
  # ──────────────────────────────────────────────────────────────────────────────
22
  print("=" * 60)
23
  print(" Loading openbmb/MiniCPM5-1B (FP16/BF16 Transformers)")
 
27
 
28
  model = AutoModelForCausalLM.from_pretrained(
29
  MODEL_ID,
30
+ torch_dtype = "auto",
31
+ device_map = "auto",
32
  trust_remote_code = True,
33
  )
34
  model.eval()
 
36
 
37
 
38
  # ──────────────────────────────────────────────────────────────────────────────
39
+ # TTS β€” persistent event loop to avoid asyncio warnings
40
  # ──────────────────────────────────────────────────────────────────────────────
41
+ _tts_loop = None
42
+
43
+ def _get_or_create_loop():
44
+ global _tts_loop
45
+ if _tts_loop is None or _tts_loop.is_closed():
46
+ _tts_loop = asyncio.new_event_loop()
47
+ return _tts_loop
48
+
49
  async def _tts_async(text: str, path: str):
50
  await edge_tts.Communicate(text, TTS_VOICE).save(path)
51
 
52
  def text_to_speech(text: str) -> str:
53
+ loop = _get_or_create_loop()
54
+ loop.run_until_complete(_tts_async(text, AUDIO_OUT))
55
  return AUDIO_OUT
56
 
57
 
58
  # ──────────────────────────────────────────────────────────────────────────────
59
+ # INFERENCE (Gradio 5 message format: list of dicts with 'role' + 'content')
60
  # ──────────────────────────────────────────────────────────────────────────────
61
  @spaces.GPU(duration=120)
62
  def chat_and_speak(
63
  user_message: str,
64
+ history: list, # list of {"role": ..., "content": ...} dicts
65
  system_prompt: str,
66
  max_new_tokens: int,
67
  temperature: float,
68
  enable_thinking: bool,
69
  enable_tts: bool,
70
  ):
71
+ # Build messages list for the model
72
  messages = [{"role": "system", "content": system_prompt}]
73
+
74
+ # history is already in the correct format (list of role/content dicts)
75
+ messages.extend(history)
76
  messages.append({"role": "user", "content": user_message})
77
 
78
+ # Tokenize
79
  inputs = tokenizer.apply_chat_template(
80
  messages,
81
+ tokenize = True,
82
+ add_generation_prompt = True,
83
+ enable_thinking = enable_thinking,
84
+ return_dict = True,
85
+ return_tensors = "pt",
86
  ).to(model.device)
87
 
88
+ # Setup streaming
89
  streamer = TextIteratorStreamer(
90
  tokenizer,
91
+ skip_prompt = True,
92
+ skip_special_tokens = True,
93
  )
94
 
95
  gen_kwargs = dict(
96
  **inputs,
97
+ max_new_tokens = max_new_tokens,
98
+ temperature = temperature,
99
+ do_sample = temperature > 0,
100
+ streamer = streamer,
101
+ pad_token_id = tokenizer.eos_token_id,
102
  )
103
 
 
104
  thread = Thread(target=model.generate, kwargs=gen_kwargs)
105
  thread.start()
106
 
107
+ # Stream tokens
108
  partial_reply = ""
109
+ current_history = history + [
110
+ {"role": "user", "content": user_message},
111
+ {"role": "assistant", "content": ""},
112
+ ]
113
 
114
  for new_token in streamer:
115
  partial_reply += new_token
116
+ current_history[-1]["content"] = partial_reply
117
+ yield current_history, current_history, None
118
 
119
  thread.join()
120
 
121
+ # Clean <think>...</think> blocks
122
  clean_reply = re.sub(
123
  r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
124
  ).strip()
125
 
126
+ current_history[-1]["content"] = clean_reply
127
 
128
  # TTS
129
  audio_path = None
 
140
  # ──────────────────────────────────────────────────────────────────────────────
141
  # GRADIO UI
142
  # ──────────────────────────────────────────────────────────────────────────────
143
+ with gr.Blocks(title="MiniCPM5-1B Chat") as demo:
 
 
144
 
145
  demo.theme = gr.themes.Soft()
146
  demo.css = "#chatbot{height:500px} footer{display:none!important}"
 
154
  )
155
 
156
  with gr.Row():
 
157
  with gr.Column(scale=3):
158
+ chatbot = gr.Chatbot(
159
+ elem_id="chatbot",
160
+ label="Conversation",
161
+ type="messages", # ← Gradio 5 explicit messages format
162
+ )
163
  with gr.Row():
164
  user_input = gr.Textbox(
165
  placeholder="Type your message …",
 
173
  type="filepath", autoplay=True,
174
  )
175
 
 
176
  with gr.Column(scale=1):
177
  gr.Markdown("### βš™οΈ Settings")
178
  system_prompt = gr.Textbox(
 
208
 
209
  state = gr.State([])
210
 
 
211
  for trigger in [send_btn.click, user_input.submit]:
212
  trigger(
213
  fn = chat_and_speak,
 
226
  demo.queue(max_size=5).launch(
227
  server_name="0.0.0.0",
228
  server_port=7860,
 
 
229
  )