Spaces:
Sleeping
Sleeping
Upload 2 files
Browse files- app.py +209 -114
- requirements.txt +4 -3
app.py
CHANGED
|
@@ -1,39 +1,45 @@
|
|
| 1 |
import os
|
| 2 |
-
import re
|
| 3 |
import threading
|
| 4 |
-
from typing import Any
|
| 5 |
|
| 6 |
import gradio as gr
|
| 7 |
-
|
|
|
|
| 8 |
|
| 9 |
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
|
|
|
|
|
|
|
|
|
| 13 |
N_THREADS = int(os.getenv("N_THREADS", str(max(1, os.cpu_count() or 1))))
|
| 14 |
-
MAX_TOKENS = int(os.getenv("MAX_TOKENS", "768"))
|
| 15 |
DEFAULT_SYSTEM_PROMPT = os.getenv(
|
| 16 |
"SYSTEM_PROMPT",
|
| 17 |
"You are a helpful assistant. Keep answers clear and concise.",
|
| 18 |
)
|
| 19 |
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
PRESETS: dict[str, dict[str, Any]] = {
|
| 23 |
"Math": {
|
| 24 |
-
"system": "You are a careful math tutor.
|
| 25 |
"prompt": "Solve: If 2x^2 - 7x + 3 = 0, what are the real solutions?",
|
| 26 |
"thinking": True,
|
| 27 |
-
"sample_reasoning": "
|
| 28 |
"sample_answer": "The real solutions are x = 3 and x = 1/2.",
|
| 29 |
},
|
| 30 |
"Coding": {
|
| 31 |
"system": "You are a Python assistant. Prefer short, readable code.",
|
| 32 |
"prompt": "Write a Python function that merges two sorted lists into one sorted list.",
|
| 33 |
"thinking": True,
|
| 34 |
-
"sample_reasoning": "Use
|
| 35 |
"sample_answer": "Here is a compact merge function plus a tiny example.",
|
| 36 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 37 |
"Function calling style": {
|
| 38 |
"system": "You are an assistant that plans tool use when it helps. If a tool would help, say what tool you would call and with which arguments.",
|
| 39 |
"prompt": "Pretend you have tools. For 18.75 * 42 - 199 and converting 12 km to miles, explain which tool calls you would make, then give the result.",
|
|
@@ -41,52 +47,47 @@ PRESETS: dict[str, dict[str, Any]] = {
|
|
| 41 |
"sample_reasoning": "I would use a calculator tool for the arithmetic and a unit-conversion tool for the distance conversion.",
|
| 42 |
"sample_answer": "Calculator(18.75 * 42 - 199) -> 588.5\nConvert(12 km -> miles) -> about 7.46 miles",
|
| 43 |
},
|
| 44 |
-
"Structured output": {
|
| 45 |
-
"system": "Return compact JSON and avoid extra commentary.",
|
| 46 |
-
"prompt": "Extract JSON from: Call Mina by Friday, priority high, budget about $2400, topic is launch video edits.",
|
| 47 |
-
"thinking": False,
|
| 48 |
-
"sample_reasoning": "Thinking is off here so the output stays short and machine-friendly.",
|
| 49 |
-
"sample_answer": '{"person":"Mina","deadline":"Friday","priority":"high","budget_usd":2400,"topic":"launch video edits"}',
|
| 50 |
-
},
|
| 51 |
"Creative writing": {
|
| 52 |
"system": "Write vivid, tight prose.",
|
| 53 |
"prompt": "Write a two-sentence opening for a sci-fi heist story set on a drifting museum ship.",
|
| 54 |
"thinking": False,
|
| 55 |
-
"sample_reasoning": "
|
| 56 |
"sample_answer": "By the time the museum ship crossed into the dead zone, every priceless relic aboard had started broadcasting a heartbeat. Nia took that as her cue to cut the lights and steal the one artifact already trying to escape.",
|
| 57 |
},
|
| 58 |
}
|
| 59 |
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 75 |
)
|
| 76 |
-
|
|
|
|
| 77 |
|
| 78 |
|
| 79 |
-
def
|
| 80 |
-
|
| 81 |
-
match = THINK_RE.search(text)
|
| 82 |
-
if not match:
|
| 83 |
-
return "", text
|
| 84 |
-
reasoning = match.group(1).strip()
|
| 85 |
-
answer = match.group(2).strip()
|
| 86 |
-
return reasoning, answer
|
| 87 |
|
| 88 |
|
| 89 |
-
def load_preset(name
|
| 90 |
preset = PRESETS[name]
|
| 91 |
return (
|
| 92 |
preset["system"],
|
|
@@ -97,78 +98,173 @@ def load_preset(name: str):
|
|
| 97 |
)
|
| 98 |
|
| 99 |
|
| 100 |
-
def
|
| 101 |
-
|
| 102 |
-
system_prompt: str,
|
| 103 |
-
thinking: bool,
|
| 104 |
-
model_history: list[dict[str, str]],
|
| 105 |
-
reasoning_chat: list[dict[str, str]],
|
| 106 |
-
answer_chat: list[dict[str, str]],
|
| 107 |
-
):
|
| 108 |
-
message = (message or "").strip()
|
| 109 |
-
if not message:
|
| 110 |
-
return reasoning_chat, answer_chat, model_history, ""
|
| 111 |
|
| 112 |
-
model_history = model_history or []
|
| 113 |
-
reasoning_chat = reasoning_chat or []
|
| 114 |
-
answer_chat = answer_chat or []
|
| 115 |
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
]
|
| 122 |
|
| 123 |
-
temperature = 0.6 if thinking else 0.7
|
| 124 |
-
top_p = 0.95 if thinking else 0.8
|
| 125 |
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
temperature=temperature,
|
| 131 |
-
top_p=top_p,
|
| 132 |
-
top_k=20,
|
| 133 |
-
presence_penalty=1.5,
|
| 134 |
-
max_tokens=MAX_TOKENS,
|
| 135 |
-
)
|
| 136 |
-
raw_text = result["choices"][0]["message"]["content"]
|
| 137 |
-
reasoning, answer = split_reasoning_and_answer(raw_text)
|
| 138 |
-
if not answer:
|
| 139 |
-
answer = raw_text.strip() or "(empty response)"
|
| 140 |
-
if not reasoning:
|
| 141 |
-
reasoning = "(No separate <think> block was returned for this turn.)"
|
| 142 |
-
except Exception as exc:
|
| 143 |
-
reasoning = ""
|
| 144 |
-
answer = f"Error while running the local GGUF model: {exc}"
|
| 145 |
|
| 146 |
-
reasoning_chat = reasoning_chat + [
|
| 147 |
-
{"role": "user", "content": message},
|
| 148 |
-
{"role": "assistant", "content": reasoning},
|
| 149 |
-
]
|
| 150 |
-
answer_chat = answer_chat + [
|
| 151 |
-
{"role": "user", "content": message},
|
| 152 |
-
{"role": "assistant", "content": answer},
|
| 153 |
-
]
|
| 154 |
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
]
|
| 160 |
|
| 161 |
-
|
|
|
|
|
|
|
|
|
|
| 162 |
|
|
|
|
| 163 |
|
| 164 |
-
def clear_all():
|
| 165 |
-
return [], [], [], ""
|
| 166 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 167 |
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
gr.Markdown(
|
| 170 |
-
"# Local
|
| 171 |
-
f"Running a local
|
|
|
|
| 172 |
)
|
| 173 |
|
| 174 |
with gr.Row():
|
|
@@ -185,12 +281,11 @@ with gr.Blocks(title="Local GGUF Split Reasoning Chat") as demo:
|
|
| 185 |
lines=3,
|
| 186 |
)
|
| 187 |
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
)
|
| 194 |
|
| 195 |
with gr.Row():
|
| 196 |
sample_reasoning = gr.Textbox(
|
|
@@ -223,12 +318,12 @@ with gr.Blocks(title="Local GGUF Split Reasoning Chat") as demo:
|
|
| 223 |
)
|
| 224 |
|
| 225 |
send_btn.click(
|
| 226 |
-
fn=
|
| 227 |
inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
|
| 228 |
outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
|
| 229 |
)
|
| 230 |
user_input.submit(
|
| 231 |
-
fn=
|
| 232 |
inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
|
| 233 |
outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
|
| 234 |
)
|
|
|
|
| 1 |
import os
|
|
|
|
| 2 |
import threading
|
|
|
|
| 3 |
|
| 4 |
import gradio as gr
|
| 5 |
+
import torch
|
| 6 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
|
| 7 |
|
| 8 |
|
| 9 |
+
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
|
| 10 |
+
|
| 11 |
+
MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen3-0.6B")
|
| 12 |
+
MAX_NEW_TOKENS = int(os.getenv("MAX_NEW_TOKENS", "256"))
|
| 13 |
+
MAX_INPUT_TOKENS = int(os.getenv("MAX_INPUT_TOKENS", "1536"))
|
| 14 |
+
MAX_HISTORY_TURNS = int(os.getenv("MAX_HISTORY_TURNS", "3"))
|
| 15 |
N_THREADS = int(os.getenv("N_THREADS", str(max(1, os.cpu_count() or 1))))
|
|
|
|
| 16 |
DEFAULT_SYSTEM_PROMPT = os.getenv(
|
| 17 |
"SYSTEM_PROMPT",
|
| 18 |
"You are a helpful assistant. Keep answers clear and concise.",
|
| 19 |
)
|
| 20 |
|
| 21 |
+
PRESETS = {
|
|
|
|
|
|
|
| 22 |
"Math": {
|
| 23 |
+
"system": "You are a careful math tutor. Think through the problem, then give a short final answer.",
|
| 24 |
"prompt": "Solve: If 2x^2 - 7x + 3 = 0, what are the real solutions?",
|
| 25 |
"thinking": True,
|
| 26 |
+
"sample_reasoning": "The discriminant is 49 - 24 = 25, so the roots are easy to compute with the quadratic formula.",
|
| 27 |
"sample_answer": "The real solutions are x = 3 and x = 1/2.",
|
| 28 |
},
|
| 29 |
"Coding": {
|
| 30 |
"system": "You are a Python assistant. Prefer short, readable code.",
|
| 31 |
"prompt": "Write a Python function that merges two sorted lists into one sorted list.",
|
| 32 |
"thinking": True,
|
| 33 |
+
"sample_reasoning": "Use two pointers. Compare the current elements, append the smaller one, then append the leftovers.",
|
| 34 |
"sample_answer": "Here is a compact merge function plus a tiny example.",
|
| 35 |
},
|
| 36 |
+
"Structured output": {
|
| 37 |
+
"system": "Return compact JSON and avoid extra commentary.",
|
| 38 |
+
"prompt": "Extract JSON from: Call Mina by Friday, priority high, budget about $2400, topic is launch video edits.",
|
| 39 |
+
"thinking": False,
|
| 40 |
+
"sample_reasoning": "Reasoning is disabled here so the output stays short and machine-friendly.",
|
| 41 |
+
"sample_answer": '{"person":"Mina","deadline":"Friday","priority":"high","budget_usd":2400,"topic":"launch video edits"}',
|
| 42 |
+
},
|
| 43 |
"Function calling style": {
|
| 44 |
"system": "You are an assistant that plans tool use when it helps. If a tool would help, say what tool you would call and with which arguments.",
|
| 45 |
"prompt": "Pretend you have tools. For 18.75 * 42 - 199 and converting 12 km to miles, explain which tool calls you would make, then give the result.",
|
|
|
|
| 47 |
"sample_reasoning": "I would use a calculator tool for the arithmetic and a unit-conversion tool for the distance conversion.",
|
| 48 |
"sample_answer": "Calculator(18.75 * 42 - 199) -> 588.5\nConvert(12 km -> miles) -> about 7.46 miles",
|
| 49 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
"Creative writing": {
|
| 51 |
"system": "Write vivid, tight prose.",
|
| 52 |
"prompt": "Write a two-sentence opening for a sci-fi heist story set on a drifting museum ship.",
|
| 53 |
"thinking": False,
|
| 54 |
+
"sample_reasoning": "Reasoning is disabled for a faster clean draft.",
|
| 55 |
"sample_answer": "By the time the museum ship crossed into the dead zone, every priceless relic aboard had started broadcasting a heartbeat. Nia took that as her cue to cut the lights and steal the one artifact already trying to escape.",
|
| 56 |
},
|
| 57 |
}
|
| 58 |
|
| 59 |
+
|
| 60 |
+
torch.set_num_threads(N_THREADS)
|
| 61 |
+
try:
|
| 62 |
+
torch.set_num_interop_threads(max(1, min(2, N_THREADS)))
|
| 63 |
+
except RuntimeError:
|
| 64 |
+
pass
|
| 65 |
+
|
| 66 |
+
_tokenizer = None
|
| 67 |
+
_model = None
|
| 68 |
+
_load_lock = threading.Lock()
|
| 69 |
+
_generate_lock = threading.Lock()
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def get_model():
|
| 73 |
+
global _tokenizer, _model
|
| 74 |
+
if _model is None or _tokenizer is None:
|
| 75 |
+
with _load_lock:
|
| 76 |
+
if _model is None or _tokenizer is None:
|
| 77 |
+
_tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
|
| 78 |
+
_model = AutoModelForCausalLM.from_pretrained(
|
| 79 |
+
MODEL_ID,
|
| 80 |
+
torch_dtype=torch.float32,
|
| 81 |
)
|
| 82 |
+
_model.eval()
|
| 83 |
+
return _tokenizer, _model
|
| 84 |
|
| 85 |
|
| 86 |
+
def clone_messages(messages):
|
| 87 |
+
return [dict(item) for item in (messages or [])]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
|
| 90 |
+
def load_preset(name):
|
| 91 |
preset = PRESETS[name]
|
| 92 |
return (
|
| 93 |
preset["system"],
|
|
|
|
| 98 |
)
|
| 99 |
|
| 100 |
|
| 101 |
+
def clear_all():
|
| 102 |
+
return [], [], [], ""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
|
|
|
|
|
|
|
|
|
|
| 104 |
|
| 105 |
+
def strip_non_think_specials(text):
|
| 106 |
+
text = text or ""
|
| 107 |
+
for token in ["<|im_end|>", "<|endoftext|>", "<|end▁of▁sentence|>"]:
|
| 108 |
+
text = text.replace(token, "")
|
| 109 |
+
return text
|
|
|
|
| 110 |
|
|
|
|
|
|
|
| 111 |
|
| 112 |
+
def final_cleanup(text):
|
| 113 |
+
text = strip_non_think_specials(text)
|
| 114 |
+
text = text.replace("<think>", "").replace("</think>", "")
|
| 115 |
+
return text.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
+
def split_stream_text(raw_text, thinking):
|
| 119 |
+
raw_text = strip_non_think_specials(raw_text)
|
| 120 |
+
if not thinking:
|
| 121 |
+
return "", final_cleanup(raw_text), False
|
|
|
|
| 122 |
|
| 123 |
+
raw_text = raw_text.replace("<think>", "")
|
| 124 |
+
if "</think>" in raw_text:
|
| 125 |
+
reasoning, answer = raw_text.split("</think>", 1)
|
| 126 |
+
return reasoning.strip(), answer.strip(), True
|
| 127 |
|
| 128 |
+
return raw_text.strip(), "", False
|
| 129 |
|
|
|
|
|
|
|
| 130 |
|
| 131 |
+
def respond_stream(
|
| 132 |
+
message,
|
| 133 |
+
system_prompt,
|
| 134 |
+
thinking,
|
| 135 |
+
model_history,
|
| 136 |
+
reasoning_chat,
|
| 137 |
+
answer_chat,
|
| 138 |
+
):
|
| 139 |
+
message = (message or "").strip()
|
| 140 |
+
if not message:
|
| 141 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history or []), ""
|
| 142 |
+
return
|
| 143 |
+
|
| 144 |
+
model_history = list(model_history or [])
|
| 145 |
+
reasoning_chat = clone_messages(reasoning_chat)
|
| 146 |
+
answer_chat = clone_messages(answer_chat)
|
| 147 |
+
|
| 148 |
+
reasoning_chat.append({"role": "user", "content": message})
|
| 149 |
+
reasoning_chat.append(
|
| 150 |
+
{
|
| 151 |
+
"role": "assistant",
|
| 152 |
+
"content": "(thinking...)" if thinking else "(reasoning disabled)",
|
| 153 |
+
}
|
| 154 |
+
)
|
| 155 |
+
answer_chat.append({"role": "user", "content": message})
|
| 156 |
+
answer_chat.append({"role": "assistant", "content": ""})
|
| 157 |
+
|
| 158 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
|
| 159 |
|
| 160 |
+
try:
|
| 161 |
+
tokenizer, model = get_model()
|
| 162 |
+
short_history = model_history[-2 * MAX_HISTORY_TURNS :]
|
| 163 |
+
messages = [
|
| 164 |
+
{"role": "system", "content": (system_prompt or "").strip() or DEFAULT_SYSTEM_PROMPT},
|
| 165 |
+
*short_history,
|
| 166 |
+
{"role": "user", "content": message},
|
| 167 |
+
]
|
| 168 |
+
|
| 169 |
+
prompt = tokenizer.apply_chat_template(
|
| 170 |
+
messages,
|
| 171 |
+
tokenize=False,
|
| 172 |
+
add_generation_prompt=True,
|
| 173 |
+
enable_thinking=thinking,
|
| 174 |
+
)
|
| 175 |
+
inputs = tokenizer(prompt, return_tensors="pt")
|
| 176 |
+
input_ids = inputs["input_ids"][:, -MAX_INPUT_TOKENS:]
|
| 177 |
+
attention_mask = inputs["attention_mask"][:, -MAX_INPUT_TOKENS:]
|
| 178 |
+
|
| 179 |
+
streamer = TextIteratorStreamer(
|
| 180 |
+
tokenizer,
|
| 181 |
+
skip_prompt=True,
|
| 182 |
+
skip_special_tokens=False,
|
| 183 |
+
clean_up_tokenization_spaces=False,
|
| 184 |
+
timeout=None,
|
| 185 |
+
)
|
| 186 |
+
|
| 187 |
+
generation_kwargs = {
|
| 188 |
+
"input_ids": input_ids,
|
| 189 |
+
"attention_mask": attention_mask,
|
| 190 |
+
"max_new_tokens": MAX_NEW_TOKENS,
|
| 191 |
+
"do_sample": True,
|
| 192 |
+
"temperature": 0.6 if thinking else 0.7,
|
| 193 |
+
"top_p": 0.95 if thinking else 0.8,
|
| 194 |
+
"top_k": 20,
|
| 195 |
+
"pad_token_id": tokenizer.eos_token_id,
|
| 196 |
+
"streamer": streamer,
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
generation_error = {}
|
| 200 |
+
|
| 201 |
+
def run_generation():
|
| 202 |
+
try:
|
| 203 |
+
with _generate_lock:
|
| 204 |
+
model.generate(**generation_kwargs)
|
| 205 |
+
except Exception as exc:
|
| 206 |
+
generation_error["message"] = str(exc)
|
| 207 |
+
streamer.on_finalized_text("", stream_end=True)
|
| 208 |
+
|
| 209 |
+
thread = threading.Thread(target=run_generation, daemon=True)
|
| 210 |
+
thread.start()
|
| 211 |
+
|
| 212 |
+
raw_text = ""
|
| 213 |
+
saw_end_think = False
|
| 214 |
+
|
| 215 |
+
for chunk in streamer:
|
| 216 |
+
raw_text += chunk
|
| 217 |
+
reasoning_text, answer_text, saw_end_now = split_stream_text(raw_text, thinking)
|
| 218 |
+
saw_end_think = saw_end_think or saw_end_now
|
| 219 |
+
|
| 220 |
+
if thinking:
|
| 221 |
+
if saw_end_think:
|
| 222 |
+
reasoning_chat[-1]["content"] = reasoning_text or "(no reasoning text returned)"
|
| 223 |
+
else:
|
| 224 |
+
reasoning_chat[-1]["content"] = reasoning_text or "(thinking...)"
|
| 225 |
+
else:
|
| 226 |
+
reasoning_chat[-1]["content"] = "(reasoning disabled)"
|
| 227 |
+
|
| 228 |
+
answer_chat[-1]["content"] = answer_text
|
| 229 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
|
| 230 |
+
|
| 231 |
+
thread.join()
|
| 232 |
+
|
| 233 |
+
if generation_error:
|
| 234 |
+
reasoning_chat[-1]["content"] = ""
|
| 235 |
+
answer_chat[-1]["content"] = f"Error while running the local CPU model: {generation_error['message']}"
|
| 236 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
|
| 237 |
+
return
|
| 238 |
+
|
| 239 |
+
reasoning_text, answer_text, saw_end_think = split_stream_text(raw_text, thinking)
|
| 240 |
+
if thinking and not saw_end_think:
|
| 241 |
+
reasoning_text = ""
|
| 242 |
+
answer_text = final_cleanup(raw_text)
|
| 243 |
+
|
| 244 |
+
if thinking:
|
| 245 |
+
reasoning_chat[-1]["content"] = reasoning_text or "(no reasoning text returned)"
|
| 246 |
+
else:
|
| 247 |
+
reasoning_chat[-1]["content"] = "(reasoning disabled)"
|
| 248 |
+
|
| 249 |
+
answer_chat[-1]["content"] = answer_text or "(empty response)"
|
| 250 |
+
model_history = short_history + [
|
| 251 |
+
{"role": "user", "content": message},
|
| 252 |
+
{"role": "assistant", "content": answer_chat[-1]["content"]},
|
| 253 |
+
]
|
| 254 |
+
|
| 255 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
|
| 256 |
+
|
| 257 |
+
except Exception as exc:
|
| 258 |
+
reasoning_chat[-1]["content"] = ""
|
| 259 |
+
answer_chat[-1]["content"] = f"Error while preparing the local CPU model: {exc}"
|
| 260 |
+
yield clone_messages(reasoning_chat), clone_messages(answer_chat), list(model_history), ""
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
with gr.Blocks(title="Local CPU split-reasoning chat") as demo:
|
| 264 |
gr.Markdown(
|
| 265 |
+
"# Local CPU split-reasoning chat\n"
|
| 266 |
+
f"Running a local safetensors model on CPU from `{MODEL_ID}`. No GGUF and no external inference provider.\n\n"
|
| 267 |
+
"The first request downloads the model, so the cold start is slower."
|
| 268 |
)
|
| 269 |
|
| 270 |
with gr.Row():
|
|
|
|
| 281 |
lines=3,
|
| 282 |
)
|
| 283 |
|
| 284 |
+
user_input = gr.Textbox(
|
| 285 |
+
label="Your message",
|
| 286 |
+
value=PRESETS["Math"]["prompt"],
|
| 287 |
+
lines=4,
|
| 288 |
+
)
|
|
|
|
| 289 |
|
| 290 |
with gr.Row():
|
| 291 |
sample_reasoning = gr.Textbox(
|
|
|
|
| 318 |
)
|
| 319 |
|
| 320 |
send_btn.click(
|
| 321 |
+
fn=respond_stream,
|
| 322 |
inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
|
| 323 |
outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
|
| 324 |
)
|
| 325 |
user_input.submit(
|
| 326 |
+
fn=respond_stream,
|
| 327 |
inputs=[user_input, system_prompt, thinking, model_history_state, reasoning_bot, answer_bot],
|
| 328 |
outputs=[reasoning_bot, answer_bot, model_history_state, user_input],
|
| 329 |
)
|
requirements.txt
CHANGED
|
@@ -1,3 +1,4 @@
|
|
| 1 |
-
gradio
|
| 2 |
-
|
| 3 |
-
|
|
|
|
|
|
| 1 |
+
gradio>=5.0.0
|
| 2 |
+
transformers>=4.51.0
|
| 3 |
+
torch>=2.2.0
|
| 4 |
+
safetensors>=0.4.0
|