eleeter commited on
Commit
d520eee
Β·
verified Β·
1 Parent(s): e204011

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +24 -29
app.py CHANGED
@@ -5,40 +5,35 @@ from llama_cpp import Llama
5
  from huggingface_hub import hf_hub_download
6
 
7
  # ──────────────────────────────────────────────
8
- # MODEL LOADING (cached to /tmp to skip re-download on warm restarts)
 
 
9
  # ──────────────────────────────────────────────
10
 
11
  MODEL_CACHE = "/tmp/qwen_coder.gguf"
12
 
13
  print("Checking model cache...")
14
- if not os.path.exists(MODEL_CACHE):
15
- print("Downloading Q4_K_M model (first run only)...")
16
  downloaded_path = hf_hub_download(
17
- repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
18
- filename="qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", # ~2x faster than Q8
19
  )
20
- # os.rename() fails across filesystems (e.g. HF cache β†’ /tmp are different partitions)
21
- # shutil.copy2 works reliably across any filesystems
22
- print(f"Copying model to cache: {downloaded_path} β†’ {MODEL_CACHE}")
23
  shutil.copy2(downloaded_path, MODEL_CACHE)
24
  print("Model cached.")
25
  else:
26
- print("Model found in cache, skipping download.")
27
 
28
- # Verify the file actually exists and has size before loading
29
- if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) == 0:
30
- raise RuntimeError(f"Model cache file missing or empty at {MODEL_CACHE}")
31
-
32
- print(f"Model file size: {os.path.getsize(MODEL_CACHE) / 1e9:.2f} GB")
33
- print("Loading model into memory...")
34
  llm = Llama(
35
  model_path=MODEL_CACHE,
36
- n_ctx=512, # Reduced from 1024 β€” context scaling costs CPU cycles
37
- n_threads=2, # Matches HuggingFace free tier vCPU count
38
- n_batch=128, # Lower batch = less memory pressure on CPU
39
- n_gpu_layers=0, # No GPU on free tier
40
- use_mlock=False, # Don't lock memory on free tier
41
- verbose=False, # Skip logging overhead
42
  )
43
  print("Model ready!")
44
 
@@ -46,16 +41,14 @@ print("Model ready!")
46
  # INFERENCE
47
  # ──────────────────────────────────────────────
48
 
49
- SYSTEM_PROMPT = "You are a helpful Minecraft coding assistant. You help with Minecraft plugin development, mods, datapacks, and commands. Be concise and practical."
50
 
51
  def respond(message, history):
52
- # Cap history to last 3 exchanges to prevent prompt bloat
53
- history = history[-3:] if history else []
54
 
55
- # Build prompt in ChatML format
56
  prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
57
  for exchange in history:
58
- user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else exchange.get("role", "")
59
  asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
60
  if user_msg:
61
  prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
@@ -66,9 +59,11 @@ def respond(message, history):
66
  response_text = ""
67
  stream = llm(
68
  prompt,
69
- max_tokens=128, # Halves worst-case wait vs 256
70
  temperature=0.7,
71
- repeat_penalty=1.1, # Prevents repetition loops that waste tokens
 
 
72
  stop=["<|im_end|>", "<|im_start|>"],
73
  stream=True,
74
  )
@@ -85,7 +80,7 @@ def respond(message, history):
85
  demo = gr.ChatInterface(
86
  fn=respond,
87
  title="⛏️ Minecraft Coding Assistant",
88
- description="Powered by Qwen2.5-Coder 1.5B Β· Q4_K_M Β· CPU optimized",
89
  examples=[
90
  "How do I create a custom recipe in a datapack?",
91
  "Write a Spigot plugin that teleports a player on command",
 
5
  from huggingface_hub import hf_hub_download
6
 
7
  # ──────────────────────────────────────────────
8
+ # MODEL: Qwen2.5-Coder 0.5B Q4_K_M
9
+ # Smallest Qwen coder model β€” ~4x faster than 1.5B on CPU
10
+ # Quality is still good for Minecraft/coding tasks
11
  # ──────────────────────────────────────────────
12
 
13
  MODEL_CACHE = "/tmp/qwen_coder.gguf"
14
 
15
  print("Checking model cache...")
16
+ if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) < 1_000_000:
17
+ print("Downloading 0.5B Q4_K_M model...")
18
  downloaded_path = hf_hub_download(
19
+ repo_id="Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF",
20
+ filename="qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
21
  )
22
+ print(f"Copying to cache...")
 
 
23
  shutil.copy2(downloaded_path, MODEL_CACHE)
24
  print("Model cached.")
25
  else:
26
+ print(f"Cache hit β€” {os.path.getsize(MODEL_CACHE)/1e9:.2f} GB")
27
 
28
+ print("Loading model...")
 
 
 
 
 
29
  llm = Llama(
30
  model_path=MODEL_CACHE,
31
+ n_ctx=256, # Tiny context β€” you don't need more for chat turns
32
+ n_threads=2, # HF free = 2 vCPUs
33
+ n_batch=32, # Small batches are faster on CPU for small models
34
+ n_gpu_layers=0,
35
+ use_mlock=False,
36
+ verbose=False,
37
  )
38
  print("Model ready!")
39
 
 
41
  # INFERENCE
42
  # ──────────────────────────────────────────────
43
 
44
+ SYSTEM_PROMPT = "You are a Minecraft coding assistant. Be brief and direct."
45
 
46
  def respond(message, history):
47
+ history = history[-2:] if history else [] # Only last 2 turns
 
48
 
 
49
  prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
50
  for exchange in history:
51
+ user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else ""
52
  asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
53
  if user_msg:
54
  prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
 
59
  response_text = ""
60
  stream = llm(
61
  prompt,
62
+ max_tokens=200,
63
  temperature=0.7,
64
+ repeat_penalty=1.1,
65
+ top_k=20, # Narrow sampling = fewer comparisons per token
66
+ top_p=0.9,
67
  stop=["<|im_end|>", "<|im_start|>"],
68
  stream=True,
69
  )
 
80
  demo = gr.ChatInterface(
81
  fn=respond,
82
  title="⛏️ Minecraft Coding Assistant",
83
+ description="Powered by Qwen2.5-Coder 0.5B Β· CPU optimized",
84
  examples=[
85
  "How do I create a custom recipe in a datapack?",
86
  "Write a Spigot plugin that teleports a player on command",