amphora commited on
Commit
c1fe670
·
verified ·
1 Parent(s): e6c9e5e

Upload 2 files

Browse files
Files changed (2) hide show
  1. app.py +209 -114
  2. requirements.txt +4 -3
app.py CHANGED
@@ -1,39 +1,45 @@
1
  import os
2
- import re
3
  import threading
4
- from typing import Any
5
 
6
  import gradio as gr
7
- from llama_cpp import Llama
 
8
 
9
 
10
- GGUF_REPO = os.getenv("GGUF_REPO", "Qwen/Qwen3-1.7B-GGUF")
11
- GGUF_FILE = os.getenv("GGUF_FILE", "Qwen3-1.7B-Q8_0.gguf")
12
- N_CTX = int(os.getenv("N_CTX", "4096"))
 
 
 
13
  N_THREADS = int(os.getenv("N_THREADS", str(max(1, os.cpu_count() or 1))))
14
- MAX_TOKENS = int(os.getenv("MAX_TOKENS", "768"))
15
  DEFAULT_SYSTEM_PROMPT = os.getenv(
16
  "SYSTEM_PROMPT",
17
  "You are a helpful assistant. Keep answers clear and concise.",
18
  )
19
 
20
- THINK_RE = re.compile(r"<think>\s*(.*?)\s*</think>\s*(.*)", re.DOTALL)
21
-
22
- PRESETS: dict[str, dict[str, Any]] = {
23
  "Math": {
24
- "system": "You are a careful math tutor. Show the reasoning clearly, then give the final answer.",
25
  "prompt": "Solve: If 2x^2 - 7x + 3 = 0, what are the real solutions?",
26
  "thinking": True,
27
- "sample_reasoning": "Factor the quadratic or use the quadratic formula. The discriminant is 25, so the roots are easy to compute.",
28
  "sample_answer": "The real solutions are x = 3 and x = 1/2.",
29
  },
30
  "Coding": {
31
  "system": "You are a Python assistant. Prefer short, readable code.",
32
  "prompt": "Write a Python function that merges two sorted lists into one sorted list.",
33
  "thinking": True,
34
- "sample_reasoning": "Use a two-pointer loop. Compare the current elements, append the smaller one, then append the leftovers.",
35
  "sample_answer": "Here is a compact merge function plus a tiny example.",
36
  },
 
 
 
 
 
 
 
37
  "Function calling style": {
38
  "system": "You are an assistant that plans tool use when it helps. If a tool would help, say what tool you would call and with which arguments.",
39
  "prompt": "Pretend you have tools. For 18.75 * 42 - 199 and converting 12 km to miles, explain which tool calls you would make, then give the result.",
@@ -41,52 +47,47 @@ PRESETS: dict[str, dict[str, Any]] = {
41
  "sample_reasoning": "I would use a calculator tool for the arithmetic and a unit-conversion tool for the distance conversion.",
42
  "sample_answer": "Calculator(18.75 * 42 - 199) -> 588.5\nConvert(12 km -> miles) -> about 7.46 miles",
43
  },
44
- "Structured output": {
45
- "system": "Return compact JSON and avoid extra commentary.",
46
- "prompt": "Extract JSON from: Call Mina by Friday, priority high, budget about $2400, topic is launch video edits.",
47
- "thinking": False,
48
- "sample_reasoning": "Thinking is off here so the output stays short and machine-friendly.",
49
- "sample_answer": '{"person":"Mina","deadline":"Friday","priority":"high","budget_usd":2400,"topic":"launch video edits"}',
50
- },
51
  "Creative writing": {
52
  "system": "Write vivid, tight prose.",
53
  "prompt": "Write a two-sentence opening for a sci-fi heist story set on a drifting museum ship.",
54
  "thinking": False,
55
- "sample_reasoning": "Thinking is off for a faster clean draft.",
56
  "sample_answer": "By the time the museum ship crossed into the dead zone, every priceless relic aboard had started broadcasting a heartbeat. Nia took that as her cue to cut the lights and steal the one artifact already trying to escape.",
57
  },
58
  }
59
 
60
- _llm = None
61
- _model_lock = threading.Lock()
62
-
63
-
64
- def get_llm() -> Llama:
65
- global _llm
66
- if _llm is None:
67
- with _model_lock:
68
- if _llm is None:
69
- _llm = Llama.from_pretrained(
70
- repo_id=GGUF_REPO,
71
- filename=GGUF_FILE,
72
- n_ctx=N_CTX,
73
- n_threads=N_THREADS,
74
- verbose=False,
 
 
 
 
 
 
 
75
  )
76
- return _llm
 
77
 
78
 
79
- def split_reasoning_and_answer(text: str) -> tuple[str, str]:
80
- text = (text or "").strip()
81
- match = THINK_RE.search(text)
82
- if not match:
83
- return "", text
84
- reasoning = match.group(1).strip()
85
- answer = match.group(2).strip()
86
- return reasoning, answer
87
 
88
 
89
- def load_preset(name: str):
90
  preset = PRESETS[name]
91
  return (
92
  preset["system"],
@@ -97,78 +98,173 @@ def load_preset(name: str):
97
  )
98
 
99
 
100
- def respond(
101
- message: str,
102
- system_prompt: str,
103
- thinking: bool,
104
- model_history: list[dict[str, str]],
105
- reasoning_chat: list[dict[str, str]],
106
- answer_chat: list[dict[str, str]],
107
- ):
108
- message = (message or "").strip()
109
- if not message:
110
- return reasoning_chat, answer_chat, model_history, ""
111
 
112
- model_history = model_history or []
113
- reasoning_chat = reasoning_chat or []
114
- answer_chat = answer_chat or []
115
 
116
- mode_suffix = "/think" if thinking else "/no_think"
117
- model_messages = [
118
- {"role": "system", "content": system_prompt.strip() or DEFAULT_SYSTEM_PROMPT},
119
- *model_history,
120
- {"role": "user", "content": f"{message} {mode_suffix}"},
121
- ]
122
 
123
- temperature = 0.6 if thinking else 0.7
124
- top_p = 0.95 if thinking else 0.8
125
 
126
- try:
127
- llm = get_llm()
128
- result = llm.create_chat_completion(
129
- messages=model_messages,
130
- temperature=temperature,
131
- top_p=top_p,
132
- top_k=20,
133
- presence_penalty=1.5,
134
- max_tokens=MAX_TOKENS,
135
- )
136
- raw_text = result["choices"][0]["message"]["content"]
137
- reasoning, answer = split_reasoning_and_answer(raw_text)
138
- if not answer:
139
- answer = raw_text.strip() or "(empty response)"
140
- if not reasoning:
141
- reasoning = "(No separate <think> block was returned for this turn.)"
142
- except Exception as exc:
143
- reasoning = ""
144
- answer = f"Error while running the local GGUF model: {exc}"
145
 
146
- reasoning_chat = reasoning_chat + [
147
- {"role": "user", "content": message},
148
- {"role": "assistant", "content": reasoning},
149
- ]
150
- answer_chat = answer_chat + [
151
- {"role": "user", "content": message},
152
- {"role": "assistant", "content": answer},
153
- ]
154
 
155
- # Keep only the final answer in model history.
156
- model_history = model_history + [
157
- {"role": "user", "content": message},
158
- {"role": "assistant", "content": answer},
159
- ]
160
 
161
- return reasoning_chat, answer_chat, model_history, ""
 
 
 
162
 
 
163
 
164
- def clear_all():
165
- return [], [], [], ""
166
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
 
168
- with gr.Blocks(title="Local GGUF Split Reasoning Chat") as demo:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  gr.Markdown(
170
- "# Local GGUF split-reasoning chat\n"
171
- f"Running a local GGUF on CPU from `{GGUF_REPO}` / `{GGUF_FILE}`."
 
172
  )
173
 
174
  with gr.Row():
@@ -185,12 +281,11 @@ with gr.Blocks(title="Local GGUF Split Reasoning Chat") as demo:
185
  lines=3,
186
  )
187
 
188
- with gr.Row():
189
- user_input = gr.Textbox(
190
- label="Your message",
191
- value=PRESETS["Math"]["prompt"],
192
- lines=4,
193
- )
194
 
195
  with gr.Row():
196
  sample_reasoning = gr.Textbox(
@@ -223,12 +318,12 @@ with gr.Blocks(title="Local GGUF Split Reasoning Chat") as demo:
223
  )
224
 
225
  send_btn.click(
226
- fn=respond,
227
  inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
228
  outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
229
  )
230
  user_input.submit(
231
- fn=respond,
232
  inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
233
  outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
234
  )
 
1
  import os
 
2
  import threading
 
3
 
4
  import gradio as gr
5
+ import torch
6
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
7
 
8
 
9
+ os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
10
+
11
+ MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-0.6B")
12
+ MAX_NEW_TOKENS = int(os.getenv("MAX_NEW_TOKENS", "256"))
13
+ MAX_INPUT_TOKENS = int(os.getenv("MAX_INPUT_TOKENS", "1536"))
14
+ MAX_HISTORY_TURNS = int(os.getenv("MAX_HISTORY_TURNS", "3"))
15
  N_THREADS = int(os.getenv("N_THREADS", str(max(1, os.cpu_count() or 1))))
 
16
  DEFAULT_SYSTEM_PROMPT = os.getenv(
17
  "SYSTEM_PROMPT",
18
  "You are a helpful assistant. Keep answers clear and concise.",
19
  )
20
 
21
+ PRESETS = {
 
 
22
  "Math": {
23
+ "system": "You are a careful math tutor. Think through the problem, then give a short final answer.",
24
  "prompt": "Solve: If 2x^2 - 7x + 3 = 0, what are the real solutions?",
25
  "thinking": True,
26
+ "sample_reasoning": "The discriminant is 49 - 24 = 25, so the roots are easy to compute with the quadratic formula.",
27
  "sample_answer": "The real solutions are x = 3 and x = 1/2.",
28
  },
29
  "Coding": {
30
  "system": "You are a Python assistant. Prefer short, readable code.",
31
  "prompt": "Write a Python function that merges two sorted lists into one sorted list.",
32
  "thinking": True,
33
+ "sample_reasoning": "Use two pointers. Compare the current elements, append the smaller one, then append the leftovers.",
34
  "sample_answer": "Here is a compact merge function plus a tiny example.",
35
  },
36
+ "Structured output": {
37
+ "system": "Return compact JSON and avoid extra commentary.",
38
+ "prompt": "Extract JSON from: Call Mina by Friday, priority high, budget about $2400, topic is launch video edits.",
39
+ "thinking": False,
40
+ "sample_reasoning": "Reasoning is disabled here so the output stays short and machine-friendly.",
41
+ "sample_answer": '{"person":"Mina","deadline":"Friday","priority":"high","budget_usd":2400,"topic":"launch video edits"}',
42
+ },
43
  "Function calling style": {
44
  "system": "You are an assistant that plans tool use when it helps. If a tool would help, say what tool you would call and with which arguments.",
45
  "prompt": "Pretend you have tools. For 18.75 * 42 - 199 and converting 12 km to miles, explain which tool calls you would make, then give the result.",
 
47
  "sample_reasoning": "I would use a calculator tool for the arithmetic and a unit-conversion tool for the distance conversion.",
48
  "sample_answer": "Calculator(18.75 * 42 - 199) -> 588.5\nConvert(12 km -> miles) -> about 7.46 miles",
49
  },
 
 
 
 
 
 
 
50
  "Creative writing": {
51
  "system": "Write vivid, tight prose.",
52
  "prompt": "Write a two-sentence opening for a sci-fi heist story set on a drifting museum ship.",
53
  "thinking": False,
54
+ "sample_reasoning": "Reasoning is disabled for a faster clean draft.",
55
  "sample_answer": "By the time the museum ship crossed into the dead zone, every priceless relic aboard had started broadcasting a heartbeat. Nia took that as her cue to cut the lights and steal the one artifact already trying to escape.",
56
  },
57
  }
58
 
59
+
60
+ torch.set_num_threads(N_THREADS)
61
+ try:
62
+ torch.set_num_interop_threads(max(1, min(2, N_THREADS)))
63
+ except RuntimeError:
64
+ pass
65
+
66
+ _tokenizer = None
67
+ _model = None
68
+ _load_lock = threading.Lock()
69
+ _generate_lock = threading.Lock()
70
+
71
+
72
+ def get_model():
73
+ global _tokenizer, _model
74
+ if _model is None or _tokenizer is None:
75
+ with _load_lock:
76
+ if _model is None or _tokenizer is None:
77
+ _tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
78
+ _model = AutoModelForCausalLM.from_pretrained(
79
+ MODEL_ID,
80
+ torch_dtype=torch.float32,
81
  )
82
+ _model.eval()
83
+ return _tokenizer, _model
84
 
85
 
86
+ def clone_messages(messages):
87
+ return [dict(item) for item in (messages or [])]
 
 
 
 
 
 
88
 
89
 
90
+ def load_preset(name):
91
  preset = PRESETS[name]
92
  return (
93
  preset["system"],
 
98
  )
99
 
100
 
101
+ def clear_all():
102
+ return [], [], [], ""
 
 
 
 
 
 
 
 
 
103
 
 
 
 
104
 
105
+ def strip_non_think_specials(text):
106
+ text = text or ""
107
+ for token in ["<|im_end|>", "<|endoftext|>", "<|end▁of▁sentence|>"]:
108
+ text = text.replace(token, "")
109
+ return text
 
110
 
 
 
111
 
112
+ def final_cleanup(text):
113
+ text = strip_non_think_specials(text)
114
+ text = text.replace("<think>", "").replace("</think>", "")
115
+ return text.strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
 
 
 
 
 
 
 
 
 
117
 
118
+ def split_stream_text(raw_text, thinking):
119
+ raw_text = strip_non_think_specials(raw_text)
120
+ if not thinking:
121
+ return "", final_cleanup(raw_text), False
 
122
 
123
+ raw_text = raw_text.replace("<think>", "")
124
+ if "</think>" in raw_text:
125
+ reasoning, answer = raw_text.split("</think>", 1)
126
+ return reasoning.strip(), answer.strip(), True
127
 
128
+ return raw_text.strip(), "", False
129
 
 
 
130
 
131
+ def respond_stream(
132
+ message,
133
+ system_prompt,
134
+ thinking,
135
+ model_history,
136
+ reasoning_chat,
137
+ answer_chat,
138
+ ):
139
+ message = (message or "").strip()
140
+ if not message:
141
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history or []), ""
142
+ return
143
+
144
+ model_history = list(model_history or [])
145
+ reasoning_chat = clone_messages(reasoning_chat)
146
+ answer_chat = clone_messages(answer_chat)
147
+
148
+ reasoning_chat.append({"role": "user", "content": message})
149
+ reasoning_chat.append(
150
+ {
151
+ "role": "assistant",
152
+ "content": "(thinking...)" if thinking else "(reasoning disabled)",
153
+ }
154
+ )
155
+ answer_chat.append({"role": "user", "content": message})
156
+ answer_chat.append({"role": "assistant", "content": ""})
157
+
158
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
159
 
160
+ try:
161
+ tokenizer, model = get_model()
162
+ short_history = model_history[-2 * MAX_HISTORY_TURNS :]
163
+ messages = [
164
+ {"role": "system", "content": (system_prompt or "").strip() or DEFAULT_SYSTEM_PROMPT},
165
+ *short_history,
166
+ {"role": "user", "content": message},
167
+ ]
168
+
169
+ prompt = tokenizer.apply_chat_template(
170
+ messages,
171
+ tokenize=False,
172
+ add_generation_prompt=True,
173
+ enable_thinking=thinking,
174
+ )
175
+ inputs = tokenizer(prompt, return_tensors="pt")
176
+ input_ids = inputs["input_ids"][:, -MAX_INPUT_TOKENS:]
177
+ attention_mask = inputs["attention_mask"][:, -MAX_INPUT_TOKENS:]
178
+
179
+ streamer = TextIteratorStreamer(
180
+ tokenizer,
181
+ skip_prompt=True,
182
+ skip_special_tokens=False,
183
+ clean_up_tokenization_spaces=False,
184
+ timeout=None,
185
+ )
186
+
187
+ generation_kwargs = {
188
+ "input_ids": input_ids,
189
+ "attention_mask": attention_mask,
190
+ "max_new_tokens": MAX_NEW_TOKENS,
191
+ "do_sample": True,
192
+ "temperature": 0.6 if thinking else 0.7,
193
+ "top_p": 0.95 if thinking else 0.8,
194
+ "top_k": 20,
195
+ "pad_token_id": tokenizer.eos_token_id,
196
+ "streamer": streamer,
197
+ }
198
+
199
+ generation_error = {}
200
+
201
+ def run_generation():
202
+ try:
203
+ with _generate_lock:
204
+ model.generate(**generation_kwargs)
205
+ except Exception as exc:
206
+ generation_error["message"] = str(exc)
207
+ streamer.on_finalized_text("", stream_end=True)
208
+
209
+ thread = threading.Thread(target=run_generation, daemon=True)
210
+ thread.start()
211
+
212
+ raw_text = ""
213
+ saw_end_think = False
214
+
215
+ for chunk in streamer:
216
+ raw_text += chunk
217
+ reasoning_text, answer_text, saw_end_now = split_stream_text(raw_text, thinking)
218
+ saw_end_think = saw_end_think or saw_end_now
219
+
220
+ if thinking:
221
+ if saw_end_think:
222
+ reasoning_chat[-1]["content"] = reasoning_text or "(no reasoning text returned)"
223
+ else:
224
+ reasoning_chat[-1]["content"] = reasoning_text or "(thinking...)"
225
+ else:
226
+ reasoning_chat[-1]["content"] = "(reasoning disabled)"
227
+
228
+ answer_chat[-1]["content"] = answer_text
229
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
230
+
231
+ thread.join()
232
+
233
+ if generation_error:
234
+ reasoning_chat[-1]["content"] = ""
235
+ answer_chat[-1]["content"] = f"Error while running the local CPU model: {generation_error['message']}"
236
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
237
+ return
238
+
239
+ reasoning_text, answer_text, saw_end_think = split_stream_text(raw_text, thinking)
240
+ if thinking and not saw_end_think:
241
+ reasoning_text = ""
242
+ answer_text = final_cleanup(raw_text)
243
+
244
+ if thinking:
245
+ reasoning_chat[-1]["content"] = reasoning_text or "(no reasoning text returned)"
246
+ else:
247
+ reasoning_chat[-1]["content"] = "(reasoning disabled)"
248
+
249
+ answer_chat[-1]["content"] = answer_text or "(empty response)"
250
+ model_history = short_history + [
251
+ {"role": "user", "content": message},
252
+ {"role": "assistant", "content": answer_chat[-1]["content"]},
253
+ ]
254
+
255
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
256
+
257
+ except Exception as exc:
258
+ reasoning_chat[-1]["content"] = ""
259
+ answer_chat[-1]["content"] = f"Error while preparing the local CPU model: {exc}"
260
+ yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
261
+
262
+
263
+ with gr.Blocks(title="Local CPU split-reasoning chat") as demo:
264
  gr.Markdown(
265
+ "# Local CPU split-reasoning chat\n"
266
+ f"Running a local safetensors model on CPU from `{MODEL_ID}`. No GGUF and no external inference provider.\n\n"
267
+ "The first request downloads the model, so the cold start is slower."
268
  )
269
 
270
  with gr.Row():
 
281
  lines=3,
282
  )
283
 
284
+ user_input = gr.Textbox(
285
+ label="Your message",
286
+ value=PRESETS["Math"]["prompt"],
287
+ lines=4,
288
+ )
 
289
 
290
  with gr.Row():
291
  sample_reasoning = gr.Textbox(
 
318
  )
319
 
320
  send_btn.click(
321
+ fn=respond_stream,
322
  inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
323
  outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
324
  )
325
  user_input.submit(
326
+ fn=respond_stream,
327
  inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
328
  outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
329
  )
requirements.txt CHANGED
@@ -1,3 +1,4 @@
1
- gradio
2
- llama-cpp-python
3
- huggingface-hub
 
 
1
+ gradio>=5.0.0
2
+ transformers>=4.51.0
3
+ torch>=2.2.0
4
+ safetensors>=0.4.0