Nafiul Huda commited on
Commit
b44dd47
Β·
verified Β·
1 Parent(s): 3241eb9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +121 -232
app.py CHANGED
@@ -1,216 +1,44 @@
1
- import os
2
- import stat
3
  import asyncio
4
- import subprocess
5
- import time
6
- import urllib.request
7
- import tarfile
8
- import zipfile
9
- import json
10
  import re
 
11
 
 
12
  import gradio as gr
13
  import edge_tts
14
  import spaces
15
- from huggingface_hub import hf_hub_download
 
16
 
17
  # ──────────────────────────────────────────────────────────────────────────────
18
  # CONFIG
19
  # ──────────────────────────────────────────────────────────────────────────────
20
- MODEL_REPO = "Abiray/MiniCPM5-1B-GGUF"
21
- MODEL_FILE = "minicpm5-1b-Q6_K.gguf"
22
- MODEL_PATH = "/tmp/minicpm5-1b-Q6_K.gguf"
23
-
24
- LLAMACPP_DIR = "/tmp/llamacpp"
25
- LLAMA_SERVER = os.path.join(LLAMACPP_DIR, "llama-server")
26
- SERVER_PORT = 8765
27
- SERVER_URL = f"http://127.0.0.1:{SERVER_PORT}"
28
-
29
- TTS_VOICE = "zh-CN-XiaoyiNeural"
30
- AUDIO_OUT = "/tmp/tts_output.mp3"
31
-
32
- # Server flags β€” mirrors your Dockerfile CMD
33
- # No --n-gpu-layers: ggml-org ships no Linux CUDA binary; Ubuntu x64 is CPU-only
34
- SERVER_FLAGS = [
35
- "-m", MODEL_PATH,
36
- "--host", "127.0.0.1",
37
- "--port", str(SERVER_PORT),
38
- "-t", "4", # bump threads slightly for CPU mode
39
- "--cache-type-k", "f16",
40
- "--cache-type-v", "f16",
41
- "-c", "8024",
42
- "-n", "4096",
43
- "--log-disable",
44
- ]
45
-
46
- # ──────────────────────────────────────────────────────────────────────────────
47
- # STEP 1 β€” Download GGUF model
48
- # ──────────────────────────────────────────────────────────────────────────────
49
- def download_model():
50
- if os.path.exists(MODEL_PATH):
51
- print(f"[model] already cached β†’ {MODEL_PATH}")
52
- return
53
- print(f"[model] downloading {MODEL_FILE} …")
54
- path = hf_hub_download(
55
- repo_id = MODEL_REPO,
56
- filename = MODEL_FILE,
57
- local_dir = "/tmp",
58
- )
59
- if path != MODEL_PATH:
60
- os.rename(path, MODEL_PATH)
61
- print(f"[model] saved β†’ {MODEL_PATH}")
62
-
63
 
64
  # ──────────────────────────────────────────────────────────────────────────────
65
- # STEP 2 β€” Download pre-built llama-server from ggml-org/llama.cpp releases
66
- #
67
- # Available Linux assets in b9574 (and all recent releases):
68
- # llama-bXXXX-bin-ubuntu-x64.tar.gz ← what we want (CPU, works on ZeroGPU)
69
- # llama-bXXXX-bin-ubuntu-arm64.tar.gz
70
- # llama-bXXXX-bin-ubuntu-vulkan-x64.tar.gz
71
- # llama-bXXXX-bin-ubuntu-rocm-7.2-x64.tar.gz
72
- # llama-bXXXX-bin-ubuntu-openvino-2026.0-x64.tar.gz
73
- # llama-bXXXX-bin-ubuntu-s390x.tar.gz
74
- #
75
- # Rule: ubuntu + x64 + .tar.gz AND none of: arm, vulkan, rocm, openvino, s390x
76
  # ──────────────────────────────────────────────────────────────────────────────
77
- GH_API = "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
78
-
79
- _UBUNTU_EXCLUDES = ("arm", "vulkan", "rocm", "openvino", "s390x")
80
-
81
- def _resolve_binary_url() -> str:
82
- req = urllib.request.Request(
83
- GH_API,
84
- headers={
85
- "Accept": "application/vnd.github+json",
86
- "User-Agent": "Mozilla/5.0",
87
- },
88
- )
89
- with urllib.request.urlopen(req, timeout=30) as resp:
90
- data = json.loads(resp.read())
91
-
92
- assets = data.get("assets", [])
93
- tag = data.get("tag_name", "unknown")
94
- names = [a["name"] for a in assets]
95
- print(f"[llama.cpp] latest release: {tag} ({len(assets)} assets)")
96
-
97
- for asset in assets:
98
- name = asset["name"].lower()
99
- # Must be: ubuntu, x64, tar.gz
100
- if not ("ubuntu" in name and "x64" in name and name.endswith(".tar.gz")):
101
- continue
102
- # Must NOT be any of the specialised variants
103
- if any(exc in name for exc in _UBUNTU_EXCLUDES):
104
- continue
105
- print(f"[llama.cpp] selected asset: {asset['name']}")
106
- return asset["browser_download_url"]
107
-
108
- raise RuntimeError(
109
- f"No plain ubuntu-x64 tarball found in release {tag}.\n"
110
- f"Assets: {names}"
111
- )
112
-
113
-
114
- def download_llamacpp():
115
- if os.path.exists(LLAMA_SERVER):
116
- print(f"[llama.cpp] binary already present β†’ {LLAMA_SERVER}")
117
- return
118
-
119
- os.makedirs(LLAMACPP_DIR, exist_ok=True)
120
- url = _resolve_binary_url()
121
-
122
- archive = "/tmp/llamacpp_release.tar.gz"
123
- print("[llama.cpp] downloading …")
124
- req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
125
- with urllib.request.urlopen(req, timeout=300) as resp, \
126
- open(archive, "wb") as f:
127
- f.write(resp.read())
128
- print("[llama.cpp] extracting …")
129
-
130
- with tarfile.open(archive, "r:gz") as tf:
131
- tf.extractall(LLAMACPP_DIR)
132
- os.remove(archive)
133
-
134
- # Find llama-server anywhere in the extracted tree
135
- found = None
136
- for root, _, files in os.walk(LLAMACPP_DIR):
137
- if "llama-server" in files:
138
- found = os.path.join(root, "llama-server")
139
- break
140
-
141
- if not found:
142
- raise RuntimeError(
143
- "[llama.cpp] llama-server not found after extraction.\n"
144
- f"Extracted: {os.listdir(LLAMACPP_DIR)}"
145
- )
146
-
147
- if found != LLAMA_SERVER:
148
- os.rename(found, LLAMA_SERVER)
149
-
150
- st = os.stat(LLAMA_SERVER)
151
- os.chmod(LLAMA_SERVER, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
152
- print(f"[llama.cpp] ready β†’ {LLAMA_SERVER}")
153
-
154
-
155
- # ──────────────────────────────────────────────────────────────────────────────
156
- # STEP 3 β€” Launch llama-server subprocess
157
- # ──────────────────────────────────────────────────────────────────────────────
158
- _server_proc = None
159
-
160
- def start_server():
161
- global _server_proc
162
- if _server_proc and _server_proc.poll() is None:
163
- return # still running
164
-
165
- cmd = [LLAMA_SERVER] + SERVER_FLAGS
166
- print(f"[server] launching: {' '.join(cmd)}")
167
- _server_proc = subprocess.Popen(
168
- cmd,
169
- stdout=subprocess.DEVNULL,
170
- stderr=subprocess.DEVNULL,
171
- )
172
-
173
- for _ in range(60):
174
- try:
175
- with urllib.request.urlopen(f"{SERVER_URL}/health", timeout=2) as r:
176
- if r.status == 200:
177
- print("[server] βœ… ready")
178
- return
179
- except Exception:
180
- pass
181
- time.sleep(1)
182
-
183
- raise RuntimeError("[server] did not become healthy within 60 s")
184
-
185
 
186
- # ──────────────────────────────────────────────────────────────────────────────
187
- # STEP 4 β€” Inference
188
- # ──────────────────────────────────────────────────────────────────────────────
189
- def call_model(messages: list, max_tokens: int = 512, temperature: float = 0.7) -> str:
190
- payload = json.dumps({
191
- "model": "minicpm",
192
- "messages": messages,
193
- "max_tokens": max_tokens,
194
- "temperature": temperature,
195
- "stream": False,
196
- }).encode()
197
-
198
- req = urllib.request.Request(
199
- f"{SERVER_URL}/v1/chat/completions",
200
- data = payload,
201
- headers = {"Content-Type": "application/json"},
202
- method = "POST",
203
- )
204
- with urllib.request.urlopen(req, timeout=120) as resp:
205
- data = json.loads(resp.read())
206
 
207
- reply = data["choices"][0]["message"]["content"]
208
- reply = re.sub(r"<think>.*?</think>", "", reply, flags=re.DOTALL).strip()
209
- return reply
 
 
 
 
 
210
 
211
 
212
  # ──────────────────────────────────────────────────────────────────────────────
213
- # STEP 5 β€” TTS
214
  # ──────────────────────────────────────────────────────────────────────────────
215
  async def _tts_async(text: str, path: str):
216
  await edge_tts.Communicate(text, TTS_VOICE).save(path)
@@ -221,39 +49,79 @@ def text_to_speech(text: str) -> str:
221
 
222
 
223
  # ──────────────────────────────────────────────────────────────────────────────
224
- # BOOT
225
- # ──────────────────────────────────────────────────────────────────────────────
226
- print("=" * 60)
227
- print(" MiniCPM5-1B Q6_K | llama-server | XiaoyiNeural TTS")
228
- print("=" * 60)
229
- download_model()
230
- download_llamacpp()
231
-
232
-
233
- # ──────────────────────────────────────────────────────────────────────────────
234
- # GRADIO INFERENCE
235
  # ──────────────────────────────────────────────────────────────────────────────
236
  @spaces.GPU(duration=120)
237
  def chat_and_speak(
238
  user_message: str,
239
- history: list,
240
  system_prompt: str,
241
- max_tokens: int,
242
  temperature: float,
 
243
  enable_tts: bool,
244
  ):
245
- start_server()
246
-
247
  messages = [{"role": "system", "content": system_prompt}]
248
  for human, bot in history:
249
  messages.append({"role": "user", "content": human})
250
  messages.append({"role": "assistant", "content": bot})
251
  messages.append({"role": "user", "content": user_message})
252
 
253
- reply = call_model(messages, max_tokens, temperature)
254
- audio_path = text_to_speech(reply) if enable_tts and reply.strip() else None
255
- history = history + [[user_message, reply]]
256
- return history, history, audio_path
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
257
 
258
 
259
  def clear_all():
@@ -261,25 +129,25 @@ def clear_all():
261
 
262
 
263
  # ──────────────────────────────────────────────────────────────────────────────
264
- # UI β€” note: raw strings r"..." used for any markdown with backslashes
265
  # ──────────────────────────────────────────────────────────────────────────────
266
  with gr.Blocks(
267
  title="MiniCPM5-1B Chat",
268
- theme=gr.themes.Soft(),
269
- css="#chatbot{height:500px} footer{display:none!important}",
270
  ) as demo:
271
 
 
 
 
272
  gr.Markdown(
273
- # use r-string so \n etc. are not interpreted as escape sequences
274
  r"""
275
  # πŸ€– MiniCPM5-1B Chat
276
- **Model:** `Abiray/MiniCPM5-1B-GGUF` Β· Q6_K &nbsp;|&nbsp;
277
- **Backend:** `ggml-org/llama.cpp` ubuntu-x64 &nbsp;|&nbsp;
278
  **TTS:** `zh-CN-XiaoyiNeural`
279
  """
280
  )
281
 
282
  with gr.Row():
 
283
  with gr.Column(scale=3):
284
  chatbot = gr.Chatbot(elem_id="chatbot", label="Conversation")
285
  with gr.Row():
@@ -295,40 +163,61 @@ with gr.Blocks(
295
  type="filepath", autoplay=True,
296
  )
297
 
 
298
  with gr.Column(scale=1):
299
  gr.Markdown("### βš™οΈ Settings")
300
  system_prompt = gr.Textbox(
301
  value="You are a helpful, concise assistant.",
302
- label="System Prompt", lines=3,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
  )
304
- max_tokens = gr.Slider(64, 2048, value=512, step=64, label="Max new tokens")
305
- temperature = gr.Slider(0.1, 2.0, value=0.7, step=0.05, label="Temperature")
306
- enable_tts = gr.Checkbox(value=True, label="πŸ”Š Enable TTS")
307
  gr.Markdown(
308
  r"""
309
  ---
310
  | | |
311
  |---|---|
312
- | Model | MiniCPM5-1B Q6_K |
313
- | Context | 8024 tokens |
314
- | Max gen | 4096 tokens |
315
- | KV cache | f16 / f16 |
316
- | Backend | CPU (ubuntu-x64) |
317
  | Voice | XiaoyiNeural |
318
  """
319
  )
320
 
321
  state = gr.State([])
322
 
 
323
  for trigger in [send_btn.click, user_input.submit]:
324
  trigger(
325
  fn = chat_and_speak,
326
- inputs = [user_input, state, system_prompt,
327
- max_tokens, temperature, enable_tts],
 
 
 
328
  outputs = [chatbot, state, audio_out],
329
  ).then(lambda: "", outputs=user_input)
330
 
331
  clear_btn.click(fn=clear_all, outputs=[chatbot, state, audio_out])
332
 
 
333
  if __name__ == "__main__":
334
- demo.queue(max_size=5).launch(server_name="0.0.0.0", server_port=7860)
 
 
 
 
 
 
 
 
1
  import asyncio
 
 
 
 
 
 
2
  import re
3
+ import os
4
 
5
+ import torch
6
  import gradio as gr
7
  import edge_tts
8
  import spaces
9
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
10
+ from threading import Thread
11
 
12
  # ──────────────────────────────────────────────────────────────────────────────
13
  # CONFIG
14
  # ──────────────────────────────────────────────────────────────────────────────
15
+ MODEL_ID = "openbmb/MiniCPM5-1B"
16
+ TTS_VOICE = "zh-CN-XiaoyiNeural"
17
+ AUDIO_OUT = "/tmp/tts_output.mp3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
18
 
19
  # ──────────────────────────────────────────────────────────────────────────────
20
+ # MODEL LOAD β€” runs once at Space boot, outside @spaces.GPU
21
+ # torch_dtype="auto" β†’ bfloat16 on ZeroGPU A100, fp16 elsewhere
22
+ # device_map="auto" β†’ moves everything to CUDA when available
 
 
 
 
 
 
 
 
23
  # ──────────────────────────────────────────────────────────────────────────────
24
+ print("=" * 60)
25
+ print(" Loading openbmb/MiniCPM5-1B (FP16/BF16 Transformers)")
26
+ print("=" * 60)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
 
28
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
+ model = AutoModelForCausalLM.from_pretrained(
31
+ MODEL_ID,
32
+ torch_dtype = "auto", # bf16 on Ampere+, fp16 elsewhere
33
+ device_map = "auto", # fill GPU first, overflow to CPU
34
+ trust_remote_code = True,
35
+ )
36
+ model.eval()
37
+ print(f"[model] loaded on: {next(model.parameters()).device}")
38
 
39
 
40
  # ──────────────────────────────────────────────────────────────────────────────
41
+ # TTS β€” XiaoyiNeural (locked)
42
  # ──────────────────────────────────────────────────────────────────────────────
43
  async def _tts_async(text: str, path: str):
44
  await edge_tts.Communicate(text, TTS_VOICE).save(path)
 
49
 
50
 
51
  # ──────────────────────────────────────────────────────────────────────────────
52
+ # INFERENCE (ZeroGPU β€” GPU is live for the duration of this call)
 
 
 
 
 
 
 
 
 
 
53
  # ──────────────────────────────────────────────────────────────────────────────
54
  @spaces.GPU(duration=120)
55
  def chat_and_speak(
56
  user_message: str,
57
+ history: list, # [[human, bot], ...]
58
  system_prompt: str,
59
+ max_new_tokens: int,
60
  temperature: float,
61
+ enable_thinking: bool,
62
  enable_tts: bool,
63
  ):
64
+ # Build messages list
 
65
  messages = [{"role": "system", "content": system_prompt}]
66
  for human, bot in history:
67
  messages.append({"role": "user", "content": human})
68
  messages.append({"role": "assistant", "content": bot})
69
  messages.append({"role": "user", "content": user_message})
70
 
71
+ # Tokenise with the official MiniCPM5 chat template
72
+ inputs = tokenizer.apply_chat_template(
73
+ messages,
74
+ tokenize = True,
75
+ add_generation_prompt= True,
76
+ enable_thinking = enable_thinking, # True β†’ <think>…</think> reasoning
77
+ return_dict = True,
78
+ return_tensors = "pt",
79
+ ).to(model.device)
80
+
81
+ # Streaming via TextIteratorStreamer
82
+ streamer = TextIteratorStreamer(
83
+ tokenizer,
84
+ skip_prompt = True,
85
+ skip_special_tokens = True,
86
+ )
87
+
88
+ gen_kwargs = dict(
89
+ **inputs,
90
+ max_new_tokens = max_new_tokens,
91
+ temperature = temperature,
92
+ do_sample = temperature > 0,
93
+ streamer = streamer,
94
+ pad_token_id = tokenizer.eos_token_id,
95
+ )
96
+
97
+ # Run generation in a background thread so we can stream tokens
98
+ thread = Thread(target=model.generate, kwargs=gen_kwargs)
99
+ thread.start()
100
+
101
+ # Collect streamed tokens and yield partial updates to Gradio
102
+ partial_reply = ""
103
+ current_history = history + [[user_message, ""]]
104
+
105
+ for new_token in streamer:
106
+ partial_reply += new_token
107
+ current_history[-1][1] = partial_reply
108
+ yield current_history, current_history, None # stream to chatbot
109
+
110
+ thread.join()
111
+
112
+ # Strip <think>…</think> blocks for display & TTS
113
+ clean_reply = re.sub(
114
+ r"<think>.*?</think>", "", partial_reply, flags=re.DOTALL
115
+ ).strip()
116
+
117
+ current_history[-1][1] = clean_reply
118
+
119
+ # TTS
120
+ audio_path = None
121
+ if enable_tts and clean_reply:
122
+ audio_path = text_to_speech(clean_reply)
123
+
124
+ yield current_history, current_history, audio_path
125
 
126
 
127
  def clear_all():
 
129
 
130
 
131
  # ──────────────────────────────────────────────────────────────────────────────
132
+ # GRADIO UI
133
  # ──────────────────────────────────────────────────────────────────────────────
134
  with gr.Blocks(
135
  title="MiniCPM5-1B Chat",
 
 
136
  ) as demo:
137
 
138
+ demo.theme = gr.themes.Soft()
139
+ demo.css = "#chatbot{height:500px} footer{display:none!important}"
140
+
141
  gr.Markdown(
 
142
  r"""
143
  # πŸ€– MiniCPM5-1B Chat
144
+ **Model:** `openbmb/MiniCPM5-1B` (FP16 Β· Transformers) &nbsp;|&nbsp;
 
145
  **TTS:** `zh-CN-XiaoyiNeural`
146
  """
147
  )
148
 
149
  with gr.Row():
150
+ # ── Left: chat ──────────────────────────────────────────────────────
151
  with gr.Column(scale=3):
152
  chatbot = gr.Chatbot(elem_id="chatbot", label="Conversation")
153
  with gr.Row():
 
163
  type="filepath", autoplay=True,
164
  )
165
 
166
+ # ── Right: settings ─────────────────────────────────────────────────
167
  with gr.Column(scale=1):
168
  gr.Markdown("### βš™οΈ Settings")
169
  system_prompt = gr.Textbox(
170
  value="You are a helpful, concise assistant.",
171
+ label="System Prompt",
172
+ lines=3,
173
+ )
174
+ max_new_tokens = gr.Slider(
175
+ 64, 4096, value=512, step=64, label="Max new tokens"
176
+ )
177
+ temperature = gr.Slider(
178
+ 0.1, 2.0, value=0.7, step=0.05, label="Temperature"
179
+ )
180
+ enable_thinking = gr.Checkbox(
181
+ value=False,
182
+ label="🧠 Enable Thinking (<think> mode)",
183
+ )
184
+ enable_tts = gr.Checkbox(
185
+ value=True, label="πŸ”Š Enable TTS (XiaoyiNeural)"
186
  )
 
 
 
187
  gr.Markdown(
188
  r"""
189
  ---
190
  | | |
191
  |---|---|
192
+ | Model | MiniCPM5-1B FP16 |
193
+ | Size | 2.17 GB |
194
+ | Arch | LlamaForCausalLM |
195
+ | Backend | Transformers 5.6+ |
 
196
  | Voice | XiaoyiNeural |
197
  """
198
  )
199
 
200
  state = gr.State([])
201
 
202
+ # ── Wire send button & Enter key ────────────────────────────────────────
203
  for trigger in [send_btn.click, user_input.submit]:
204
  trigger(
205
  fn = chat_and_speak,
206
+ inputs = [
207
+ user_input, state, system_prompt,
208
+ max_new_tokens, temperature,
209
+ enable_thinking, enable_tts,
210
+ ],
211
  outputs = [chatbot, state, audio_out],
212
  ).then(lambda: "", outputs=user_input)
213
 
214
  clear_btn.click(fn=clear_all, outputs=[chatbot, state, audio_out])
215
 
216
+
217
  if __name__ == "__main__":
218
+ demo.queue(max_size=5).launch(
219
+ server_name="0.0.0.0",
220
+ server_port=7860,
221
+ theme=gr.themes.Soft(),
222
+ css="#chatbot{height:500px} footer{display:none!important}",
223
+ )