Update app.py
Browse files
app.py
CHANGED
|
@@ -5,40 +5,35 @@ from llama_cpp import Llama
|
|
| 5 |
from huggingface_hub import hf_hub_download
|
| 6 |
|
| 7 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 8 |
-
# MODEL
|
|
|
|
|
|
|
| 9 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 10 |
|
| 11 |
MODEL_CACHE = "/tmp/qwen_coder.gguf"
|
| 12 |
|
| 13 |
print("Checking model cache...")
|
| 14 |
-
if not os.path.exists(MODEL_CACHE):
|
| 15 |
-
print("Downloading Q4_K_M model
|
| 16 |
downloaded_path = hf_hub_download(
|
| 17 |
-
repo_id="Qwen/Qwen2.5-Coder-
|
| 18 |
-
filename="qwen2.5-coder-
|
| 19 |
)
|
| 20 |
-
|
| 21 |
-
# shutil.copy2 works reliably across any filesystems
|
| 22 |
-
print(f"Copying model to cache: {downloaded_path} β {MODEL_CACHE}")
|
| 23 |
shutil.copy2(downloaded_path, MODEL_CACHE)
|
| 24 |
print("Model cached.")
|
| 25 |
else:
|
| 26 |
-
print("
|
| 27 |
|
| 28 |
-
|
| 29 |
-
if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) == 0:
|
| 30 |
-
raise RuntimeError(f"Model cache file missing or empty at {MODEL_CACHE}")
|
| 31 |
-
|
| 32 |
-
print(f"Model file size: {os.path.getsize(MODEL_CACHE) / 1e9:.2f} GB")
|
| 33 |
-
print("Loading model into memory...")
|
| 34 |
llm = Llama(
|
| 35 |
model_path=MODEL_CACHE,
|
| 36 |
-
n_ctx=
|
| 37 |
-
n_threads=2,
|
| 38 |
-
n_batch=
|
| 39 |
-
n_gpu_layers=0,
|
| 40 |
-
use_mlock=False,
|
| 41 |
-
verbose=False,
|
| 42 |
)
|
| 43 |
print("Model ready!")
|
| 44 |
|
|
@@ -46,16 +41,14 @@ print("Model ready!")
|
|
| 46 |
# INFERENCE
|
| 47 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 48 |
|
| 49 |
-
SYSTEM_PROMPT = "You are a
|
| 50 |
|
| 51 |
def respond(message, history):
|
| 52 |
-
|
| 53 |
-
history = history[-3:] if history else []
|
| 54 |
|
| 55 |
-
# Build prompt in ChatML format
|
| 56 |
prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
|
| 57 |
for exchange in history:
|
| 58 |
-
user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else
|
| 59 |
asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
|
| 60 |
if user_msg:
|
| 61 |
prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
|
|
@@ -66,9 +59,11 @@ def respond(message, history):
|
|
| 66 |
response_text = ""
|
| 67 |
stream = llm(
|
| 68 |
prompt,
|
| 69 |
-
max_tokens=
|
| 70 |
temperature=0.7,
|
| 71 |
-
repeat_penalty=1.1,
|
|
|
|
|
|
|
| 72 |
stop=["<|im_end|>", "<|im_start|>"],
|
| 73 |
stream=True,
|
| 74 |
)
|
|
@@ -85,7 +80,7 @@ def respond(message, history):
|
|
| 85 |
demo = gr.ChatInterface(
|
| 86 |
fn=respond,
|
| 87 |
title="βοΈ Minecraft Coding Assistant",
|
| 88 |
-
description="Powered by Qwen2.5-Coder
|
| 89 |
examples=[
|
| 90 |
"How do I create a custom recipe in a datapack?",
|
| 91 |
"Write a Spigot plugin that teleports a player on command",
|
|
|
|
| 5 |
from huggingface_hub import hf_hub_download
|
| 6 |
|
| 7 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 8 |
+
# MODEL: Qwen2.5-Coder 0.5B Q4_K_M
|
| 9 |
+
# Smallest Qwen coder model β ~4x faster than 1.5B on CPU
|
| 10 |
+
# Quality is still good for Minecraft/coding tasks
|
| 11 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
|
| 13 |
MODEL_CACHE = "/tmp/qwen_coder.gguf"
|
| 14 |
|
| 15 |
print("Checking model cache...")
|
| 16 |
+
if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) < 1_000_000:
|
| 17 |
+
print("Downloading 0.5B Q4_K_M model...")
|
| 18 |
downloaded_path = hf_hub_download(
|
| 19 |
+
repo_id="Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF",
|
| 20 |
+
filename="qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
|
| 21 |
)
|
| 22 |
+
print(f"Copying to cache...")
|
|
|
|
|
|
|
| 23 |
shutil.copy2(downloaded_path, MODEL_CACHE)
|
| 24 |
print("Model cached.")
|
| 25 |
else:
|
| 26 |
+
print(f"Cache hit β {os.path.getsize(MODEL_CACHE)/1e9:.2f} GB")
|
| 27 |
|
| 28 |
+
print("Loading model...")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 29 |
llm = Llama(
|
| 30 |
model_path=MODEL_CACHE,
|
| 31 |
+
n_ctx=256, # Tiny context β you don't need more for chat turns
|
| 32 |
+
n_threads=2, # HF free = 2 vCPUs
|
| 33 |
+
n_batch=32, # Small batches are faster on CPU for small models
|
| 34 |
+
n_gpu_layers=0,
|
| 35 |
+
use_mlock=False,
|
| 36 |
+
verbose=False,
|
| 37 |
)
|
| 38 |
print("Model ready!")
|
| 39 |
|
|
|
|
| 41 |
# INFERENCE
|
| 42 |
# ββββββββββββββββββββββββββββββββββββββββββββββ
|
| 43 |
|
| 44 |
+
SYSTEM_PROMPT = "You are a Minecraft coding assistant. Be brief and direct."
|
| 45 |
|
| 46 |
def respond(message, history):
|
| 47 |
+
history = history[-2:] if history else [] # Only last 2 turns
|
|
|
|
| 48 |
|
|
|
|
| 49 |
prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
|
| 50 |
for exchange in history:
|
| 51 |
+
user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else ""
|
| 52 |
asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
|
| 53 |
if user_msg:
|
| 54 |
prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
|
|
|
|
| 59 |
response_text = ""
|
| 60 |
stream = llm(
|
| 61 |
prompt,
|
| 62 |
+
max_tokens=200,
|
| 63 |
temperature=0.7,
|
| 64 |
+
repeat_penalty=1.1,
|
| 65 |
+
top_k=20, # Narrow sampling = fewer comparisons per token
|
| 66 |
+
top_p=0.9,
|
| 67 |
stop=["<|im_end|>", "<|im_start|>"],
|
| 68 |
stream=True,
|
| 69 |
)
|
|
|
|
| 80 |
demo = gr.ChatInterface(
|
| 81 |
fn=respond,
|
| 82 |
title="βοΈ Minecraft Coding Assistant",
|
| 83 |
+
description="Powered by Qwen2.5-Coder 0.5B Β· CPU optimized",
|
| 84 |
examples=[
|
| 85 |
"How do I create a custom recipe in a datapack?",
|
| 86 |
"Write a Spigot plugin that teleports a player on command",
|