eleeter commited on
Commit
b2f687c
Β·
verified Β·
1 Parent(s): d520eee

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +129 -65
app.py CHANGED
@@ -1,86 +1,150 @@
1
  import os
2
- import shutil
3
  import gradio as gr
4
- from llama_cpp import Llama
5
- from huggingface_hub import hf_hub_download
6
-
7
- # ──────────────────────────────────────────────
8
- # MODEL: Qwen2.5-Coder 0.5B Q4_K_M
9
- # Smallest Qwen coder model β€” ~4x faster than 1.5B on CPU
10
- # Quality is still good for Minecraft/coding tasks
11
- # ──────────────────────────────────────────────
12
-
13
- MODEL_CACHE = "/tmp/qwen_coder.gguf"
14
-
15
- print("Checking model cache...")
16
- if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) < 1_000_000:
17
- print("Downloading 0.5B Q4_K_M model...")
18
- downloaded_path = hf_hub_download(
19
- repo_id="Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF",
20
- filename="qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
21
- )
22
- print(f"Copying to cache...")
23
- shutil.copy2(downloaded_path, MODEL_CACHE)
24
- print("Model cached.")
25
- else:
26
- print(f"Cache hit β€” {os.path.getsize(MODEL_CACHE)/1e9:.2f} GB")
27
-
28
- print("Loading model...")
29
- llm = Llama(
30
- model_path=MODEL_CACHE,
31
- n_ctx=256, # Tiny context β€” you don't need more for chat turns
32
- n_threads=2, # HF free = 2 vCPUs
33
- n_batch=32, # Small batches are faster on CPU for small models
34
- n_gpu_layers=0,
35
- use_mlock=False,
36
- verbose=False,
37
- )
38
- print("Model ready!")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
39
 
40
- # ──────────────────────────────────────────────
41
- # INFERENCE
42
- # ──────────────────────────────────────────────
 
 
 
 
43
 
44
  SYSTEM_PROMPT = "You are a Minecraft coding assistant. Be brief and direct."
45
 
46
  def respond(message, history):
47
- history = history[-2:] if history else [] # Only last 2 turns
 
 
 
 
 
 
 
 
 
 
48
 
49
- prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
50
- for exchange in history:
51
  user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else ""
52
  asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
53
  if user_msg:
54
- prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
55
  if asst_msg:
56
- prompt += f"<|im_start|>assistant\n{asst_msg}<|im_end|>\n"
57
- prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
58
-
59
- response_text = ""
60
- stream = llm(
61
- prompt,
62
- max_tokens=200,
63
- temperature=0.7,
64
- repeat_penalty=1.1,
65
- top_k=20, # Narrow sampling = fewer comparisons per token
66
- top_p=0.9,
67
- stop=["<|im_end|>", "<|im_start|>"],
68
- stream=True,
69
- )
 
 
 
 
70
 
71
- for chunk in stream:
72
- token = chunk["choices"][0]["text"]
73
- response_text += token
74
- yield response_text
 
 
 
 
 
 
75
 
76
- # ──────────────────────────────────────────────
77
- # UI
78
- # ──────────────────────────────────────────────
 
 
 
 
 
 
 
79
 
 
80
  demo = gr.ChatInterface(
81
  fn=respond,
82
  title="⛏️ Minecraft Coding Assistant",
83
- description="Powered by Qwen2.5-Coder 0.5B Β· CPU optimized",
84
  examples=[
85
  "How do I create a custom recipe in a datapack?",
86
  "Write a Spigot plugin that teleports a player on command",
 
1
  import os
2
+ import random
3
  import gradio as gr
4
+ from openai import OpenAI
5
+ from collections import defaultdict
6
+ import time
7
+
8
+ # ── Read keys from environment variables (never hardcode these!) ──
9
+ API_KEYS = {
10
+ "groq": os.environ.get("GROQ_API_KEY"),
11
+ "gemini": os.environ.get("GEMINI_API_KEY"),
12
+ "mistral": os.environ.get("MISTRAL_API_KEY"),
13
+ "cohere": os.environ.get("COHERE_API_KEY"),
14
+ "huggingface": os.environ.get("HF_API_KEY"),
15
+ }
16
+
17
+ # ── Provider configs ──
18
+ ALL_PROVIDERS = [
19
+ {
20
+ "name": "groq",
21
+ "base_url": "https://api.groq.com/openai/v1",
22
+ "model": "llama-3.1-8b-instant",
23
+ "daily_limit": 14400,
24
+ "priority": 1,
25
+ },
26
+ {
27
+ "name": "gemini",
28
+ "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/",
29
+ "model": "gemini-2.0-flash",
30
+ "daily_limit": 1500,
31
+ "priority": 2,
32
+ },
33
+ {
34
+ "name": "mistral",
35
+ "base_url": "https://api.mistral.ai/v1",
36
+ "model": "mistral-small-latest",
37
+ "daily_limit": 1400,
38
+ "priority": 3,
39
+ },
40
+ {
41
+ "name": "cohere",
42
+ "base_url": "https://api.cohere.com/compatibility/v1",
43
+ "model": "command-r-plus",
44
+ "daily_limit": 1000,
45
+ "priority": 4,
46
+ },
47
+ {
48
+ "name": "huggingface",
49
+ "base_url": "https://api-inference.huggingface.co/v1/",
50
+ "model": "Qwen/Qwen2.5-Coder-1.5B-Instruct",
51
+ "daily_limit": 1000,
52
+ "priority": 5,
53
+ },
54
+ ]
55
+
56
+ # ── Filter to only providers where key was given ──
57
+ ACTIVE_PROVIDERS = [
58
+ p for p in ALL_PROVIDERS
59
+ if API_KEYS.get(p["name"]) not in (None, "")
60
+ ]
61
+
62
+ # ── Usage tracker ──
63
+ usage_count = defaultdict(int)
64
+ last_reset = time.time()
65
+
66
+ def pick_provider():
67
+ global last_reset
68
+ if time.time() - last_reset > 86400:
69
+ usage_count.clear()
70
+ last_reset = time.time()
71
 
72
+ return min(
73
+ ACTIVE_PROVIDERS,
74
+ key=lambda p: (
75
+ usage_count[p["name"]] / p["daily_limit"],
76
+ p["priority"]
77
+ )
78
+ )
79
 
80
  SYSTEM_PROMPT = "You are a Minecraft coding assistant. Be brief and direct."
81
 
82
  def respond(message, history):
83
+ if not ACTIVE_PROVIDERS:
84
+ yield "❌ No API keys provided! Add them in your Space's Settings β†’ Variables and Secrets."
85
+ return
86
+
87
+ provider = pick_provider()
88
+ print(f"Using: {provider['name']} | uses today: {usage_count[provider['name']]}")
89
+
90
+ client = OpenAI(
91
+ api_key=API_KEYS[provider["name"]],
92
+ base_url=provider["base_url"],
93
+ )
94
 
95
+ messages = [{"role": "system", "content": SYSTEM_PROMPT}]
96
+ for exchange in history[-3:]:
97
  user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else ""
98
  asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
99
  if user_msg:
100
+ messages.append({"role": "user", "content": user_msg})
101
  if asst_msg:
102
+ messages.append({"role": "assistant", "content": asst_msg})
103
+ messages.append({"role": "user", "content": message})
104
+
105
+ try:
106
+ stream = client.chat.completions.create(
107
+ model=provider["model"],
108
+ messages=messages,
109
+ max_tokens=200,
110
+ temperature=0.7,
111
+ stream=True,
112
+ )
113
+ usage_count[provider["name"]] += 1
114
+
115
+ response_text = ""
116
+ for chunk in stream:
117
+ token = chunk.choices[0].delta.content or ""
118
+ response_text += token
119
+ yield response_text
120
 
121
+ except Exception as e:
122
+ err = str(e).lower()
123
+ if "401" in err or "unauthorized" in err or "invalid api key" in err:
124
+ yield f"❌ API key for **{provider['name']}** is invalid or expired!"
125
+ elif "429" in err or "rate limit" in err:
126
+ yield f"⚠️ **{provider['name']}** hit rate limit β€” try again in a moment"
127
+ elif "quota" in err or "exceeded" in err:
128
+ yield f"⚠️ **{provider['name']}** daily quota exceeded"
129
+ else:
130
+ yield f"⚠️ **{provider['name']}** error: {str(e)}"
131
 
132
+ # ── Startup summary ──
133
+ print("\n=== Provider Status ===")
134
+ for p in ALL_PROVIDERS:
135
+ key = API_KEYS.get(p["name"])
136
+ status = "βœ… Active" if key not in (None, "") else "❌ No key"
137
+ print(f" {p['name']:<14} {status}")
138
+ print(f"\nActive providers: {len(ACTIVE_PROVIDERS)}/{len(ALL_PROVIDERS)}")
139
+ if not ACTIVE_PROVIDERS:
140
+ print("⚠️ WARNING: No API keys found in environment!")
141
+ print("=" * 23 + "\n")
142
 
143
+ # ── UI ──
144
  demo = gr.ChatInterface(
145
  fn=respond,
146
  title="⛏️ Minecraft Coding Assistant",
147
+ description=f"Multi-provider Β· {len(ACTIVE_PROVIDERS)} active Β· ~{sum(p['daily_limit'] for p in ACTIVE_PROVIDERS):,} req/day",
148
  examples=[
149
  "How do I create a custom recipe in a datapack?",
150
  "Write a Spigot plugin that teleports a player on command",