Spaces:
Sleeping
Sleeping
Upload server.py
Browse files
server.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import uvicorn
|
| 2 |
+
import torch
|
| 3 |
+
import threading
|
| 4 |
+
import json
|
| 5 |
+
import logging
|
| 6 |
+
import sqlite3
|
| 7 |
+
import os
|
| 8 |
+
import re
|
| 9 |
+
import time
|
| 10 |
+
from typing import List, Optional, Literal, Tuple
|
| 11 |
+
from fastapi import FastAPI, HTTPException
|
| 12 |
+
from fastapi.middleware.cors import CORSMiddleware
|
| 13 |
+
from fastapi.responses import StreamingResponse
|
| 14 |
+
from pydantic import BaseModel
|
| 15 |
+
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
|
| 16 |
+
|
| 17 |
+
# --- Configuration ---
|
| 18 |
+
# HF Spaces uses port 7860 by default
|
| 19 |
+
PORT = int(os.environ.get("PORT", 7860))
|
| 20 |
+
HOST = "0.0.0.0"
|
| 21 |
+
|
| 22 |
+
MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
|
| 23 |
+
DB_FILE = "chatbot.db"
|
| 24 |
+
JSON_SOURCE = "database.json"
|
| 25 |
+
|
| 26 |
+
# --- Logging ---
|
| 27 |
+
logging.basicConfig(level=logging.INFO)
|
| 28 |
+
logger = logging.getLogger(__name__)
|
| 29 |
+
|
| 30 |
+
# --- FastAPI ---
|
| 31 |
+
app = FastAPI(title="Hebrew RAG Chatbot", version="4.0.0")
|
| 32 |
+
|
| 33 |
+
app.add_middleware(
|
| 34 |
+
CORSMiddleware,
|
| 35 |
+
allow_origins=["*"],
|
| 36 |
+
allow_credentials=True,
|
| 37 |
+
allow_methods=["*"],
|
| 38 |
+
allow_headers=["*"],
|
| 39 |
+
)
|
| 40 |
+
|
| 41 |
+
# --- Globals ---
|
| 42 |
+
model = None
|
| 43 |
+
tokenizer = None
|
| 44 |
+
device = "cpu"
|
| 45 |
+
|
| 46 |
+
# --- Hebrew Helpers ---
|
| 47 |
+
STOP_WORDS = {
|
| 48 |
+
"של", "את", "על", "עם", "כי", "זה", "או", "אם", "גם", "הוא",
|
| 49 |
+
"היא", "הם", "מה", "מי", "איך", "למה", "כל", "לא", "כן", "אני",
|
| 50 |
+
"אתה", "אנחנו", "הזה", "הזו", "שלי", "שלך", "בין", "רק", "עוד",
|
| 51 |
+
"תסביר", "לי", "ספר", "בבקשה", "היי", "שלום", "תודה", "כיצד"
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
SYNONYMS = {
|
| 55 |
+
"מתקינים": ["התקנה", "להתקין", "התקנת", "install"],
|
| 56 |
+
"התקנה": ["מתקינים", "להתקין", "setup", "installation"],
|
| 57 |
+
"מחברים": ["חיבור", "לחבר", "חיווט", "connect"],
|
| 58 |
+
"חיבור": ["מחברים", "לחבר", "כבל", "wiring"],
|
| 59 |
+
"תקלה": ["בעיה", "לא עובד", "שגיאה", "error", "bug"],
|
| 60 |
+
"קוד": ["תכנות", "פרוייקט", "קבצים", "code"],
|
| 61 |
+
"esp32": ["בקר", "מיקרובקר", "צ'יפ", "controller"],
|
| 62 |
+
"nextion": ["מסך", "תצוגה", "HMI", "screen"],
|
| 63 |
+
"מערכת": ["פרויקט", "בית חכם", "מוצר", "system"],
|
| 64 |
+
"אפליקציה": ["יישום", "app", "apk"],
|
| 65 |
+
"אימון": ["למידה", "train", "training", "ai"]
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
def clean_and_expand_query(text: str) -> List[str]:
|
| 69 |
+
text = re.sub(r'[^\w\s]', ' ', text)
|
| 70 |
+
words = text.split()
|
| 71 |
+
search_terms = set()
|
| 72 |
+
for w in words:
|
| 73 |
+
if w in STOP_WORDS or len(w) < 2: continue
|
| 74 |
+
search_terms.add(w)
|
| 75 |
+
if w in SYNONYMS:
|
| 76 |
+
for synonym in SYNONYMS[w]: search_terms.add(synonym)
|
| 77 |
+
if len(w) > 3 and w[0] in ['ה', 'ב', 'ל', 'מ']:
|
| 78 |
+
root = w[1:]
|
| 79 |
+
search_terms.add(root)
|
| 80 |
+
if root in SYNONYMS:
|
| 81 |
+
for synonym in SYNONYMS[root]: search_terms.add(synonym)
|
| 82 |
+
return list(search_terms)
|
| 83 |
+
|
| 84 |
+
# --- Database Logic ---
|
| 85 |
+
def init_db():
|
| 86 |
+
if os.path.exists(DB_FILE): os.remove(DB_FILE)
|
| 87 |
+
conn = sqlite3.connect(DB_FILE)
|
| 88 |
+
cursor = conn.cursor()
|
| 89 |
+
cursor.execute('CREATE TABLE IF NOT EXISTS knowledge (id INTEGER PRIMARY KEY, topic TEXT, content TEXT)')
|
| 90 |
+
cursor.execute('CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY, role TEXT, content TEXT)')
|
| 91 |
+
|
| 92 |
+
if os.path.exists(JSON_SOURCE):
|
| 93 |
+
try:
|
| 94 |
+
with open(JSON_SOURCE, 'r', encoding='utf-8') as f:
|
| 95 |
+
data = json.load(f)
|
| 96 |
+
for item in data:
|
| 97 |
+
if isinstance(item, dict) and 'content' in item:
|
| 98 |
+
cursor.execute('INSERT INTO knowledge (topic, content) VALUES (?, ?)',
|
| 99 |
+
(item.get('topic', 'General'), item['content']))
|
| 100 |
+
conn.commit()
|
| 101 |
+
logger.info(f"✅ SUCCESS: Loaded articles from {JSON_SOURCE}")
|
| 102 |
+
except Exception as e:
|
| 103 |
+
logger.error(f"❌ ERROR reading JSON: {e}")
|
| 104 |
+
conn.close()
|
| 105 |
+
|
| 106 |
+
def query_knowledge_base(query: str) -> List[Tuple[str, str]]:
|
| 107 |
+
conn = sqlite3.connect(DB_FILE)
|
| 108 |
+
cursor = conn.cursor()
|
| 109 |
+
search_terms = clean_and_expand_query(query)
|
| 110 |
+
results = []
|
| 111 |
+
seen_ids = set()
|
| 112 |
+
for word in search_terms:
|
| 113 |
+
cursor.execute("SELECT id, topic, content FROM knowledge WHERE content LIKE ?", (f'%{word}%',))
|
| 114 |
+
for row_id, topic, content in cursor.fetchall():
|
| 115 |
+
if row_id not in seen_ids:
|
| 116 |
+
results.append((topic, content))
|
| 117 |
+
seen_ids.add(row_id)
|
| 118 |
+
conn.close()
|
| 119 |
+
return results[:3]
|
| 120 |
+
|
| 121 |
+
def save_message(role: str, content: str):
|
| 122 |
+
try:
|
| 123 |
+
conn = sqlite3.connect(DB_FILE)
|
| 124 |
+
cursor = conn.cursor()
|
| 125 |
+
cursor.execute("INSERT INTO history (role, content) VALUES (?, ?)", (role, content))
|
| 126 |
+
conn.commit()
|
| 127 |
+
conn.close()
|
| 128 |
+
except Exception as e:
|
| 129 |
+
logger.error(f"DB Write Error: {e}")
|
| 130 |
+
|
| 131 |
+
# --- Models ---
|
| 132 |
+
class Message(BaseModel):
|
| 133 |
+
role: Literal["user", "assistant", "system"]
|
| 134 |
+
content: str
|
| 135 |
+
|
| 136 |
+
class ChatCompletionRequest(BaseModel):
|
| 137 |
+
messages: List[Message]
|
| 138 |
+
max_tokens: Optional[int] = 512
|
| 139 |
+
temperature: Optional[float] = 0.2
|
| 140 |
+
top_p: Optional[float] = 0.9
|
| 141 |
+
stream: Optional[bool] = False
|
| 142 |
+
|
| 143 |
+
# --- Startup ---
|
| 144 |
+
@app.on_event("startup")
|
| 145 |
+
async def startup_event():
|
| 146 |
+
global model, tokenizer, device
|
| 147 |
+
init_db()
|
| 148 |
+
|
| 149 |
+
logger.info(f"Loading model: {MODEL_ID}...")
|
| 150 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 151 |
+
logger.info(f"Using device: {device}")
|
| 152 |
+
|
| 153 |
+
try:
|
| 154 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
|
| 155 |
+
model = AutoModelForCausalLM.from_pretrained(
|
| 156 |
+
MODEL_ID,
|
| 157 |
+
torch_dtype=torch.float16 if device != "cpu" else torch.float32,
|
| 158 |
+
device_map=device,
|
| 159 |
+
low_cpu_mem_usage=True,
|
| 160 |
+
trust_remote_code=True
|
| 161 |
+
)
|
| 162 |
+
logger.info("Model loaded successfully!")
|
| 163 |
+
|
| 164 |
+
logger.info("Performing warmup...")
|
| 165 |
+
dummy_input = tokenizer("Hello", return_tensors="pt").to(device)
|
| 166 |
+
model.generate(**dummy_input, max_new_tokens=1)
|
| 167 |
+
logger.info("Warmup complete!")
|
| 168 |
+
|
| 169 |
+
except Exception as e:
|
| 170 |
+
logger.error(f"Failed to load: {e}")
|
| 171 |
+
raise RuntimeError("Model load failed")
|
| 172 |
+
|
| 173 |
+
def run_generation(model, tokenizer, params):
|
| 174 |
+
try:
|
| 175 |
+
model.generate(**params)
|
| 176 |
+
except Exception as e:
|
| 177 |
+
logger.error(f"Generation error: {e}")
|
| 178 |
+
|
| 179 |
+
@app.get("/health")
|
| 180 |
+
def health_check():
|
| 181 |
+
return {"status": "running", "model": MODEL_ID, "device": device}
|
| 182 |
+
|
| 183 |
+
@app.get("/")
|
| 184 |
+
def home():
|
| 185 |
+
return {"message": "Chatbot Server is Running. Use POST /v1/chat/completions"}
|
| 186 |
+
|
| 187 |
+
@app.post("/v1/chat/completions")
|
| 188 |
+
async def chat_completions(request: ChatCompletionRequest):
|
| 189 |
+
if not model or not tokenizer:
|
| 190 |
+
raise HTTPException(status_code=503, detail="Model not loaded")
|
| 191 |
+
|
| 192 |
+
last_msg = next((m for m in reversed(request.messages) if m.role == "user"), None)
|
| 193 |
+
if last_msg:
|
| 194 |
+
save_message("user", last_msg.content)
|
| 195 |
+
retrieved = query_knowledge_base(last_msg.content)
|
| 196 |
+
|
| 197 |
+
sys_prompt = (
|
| 198 |
+
"System: You are a helpful assistant for the 'Smart Home' project. "
|
| 199 |
+
"Instructions:\n"
|
| 200 |
+
"1. Answer ONLY in Hebrew (עברית).\n"
|
| 201 |
+
"2. Use the provided Context to answer the question.\n"
|
| 202 |
+
"3. If the answer is not in the context, say politely in Hebrew that you don't have the info.\n"
|
| 203 |
+
"4. Do NOT start sentences with English words."
|
| 204 |
+
)
|
| 205 |
+
|
| 206 |
+
if retrieved:
|
| 207 |
+
topics = ", ".join(sorted(list(set([i[0] for i in retrieved]))))
|
| 208 |
+
content = "\n---\n".join([i[1] for i in retrieved])
|
| 209 |
+
sys_prompt += f"\n\nContext ({topics}):\n{content}\n\nInstruction: Answer in Hebrew based on the context above."
|
| 210 |
+
else:
|
| 211 |
+
sys_prompt += "\n\nContext: No specific document found. Instruction: If this is a greeting (Shalom), reply politely in Hebrew. Otherwise, say you don't know."
|
| 212 |
+
|
| 213 |
+
if request.messages[0].role == "system":
|
| 214 |
+
request.messages[0].content = sys_prompt
|
| 215 |
+
else:
|
| 216 |
+
request.messages.insert(0, Message(role="system", content=sys_prompt))
|
| 217 |
+
|
| 218 |
+
try:
|
| 219 |
+
conversation = [msg.dict() for msg in request.messages]
|
| 220 |
+
prompt_ids = tokenizer.apply_chat_template(conversation, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(device)
|
| 221 |
+
attention_mask = torch.ones_like(prompt_ids)
|
| 222 |
+
except Exception as e:
|
| 223 |
+
raise HTTPException(status_code=400, detail=str(e))
|
| 224 |
+
|
| 225 |
+
gen_kwargs = dict(
|
| 226 |
+
inputs=prompt_ids,
|
| 227 |
+
attention_mask=attention_mask,
|
| 228 |
+
max_new_tokens=request.max_tokens,
|
| 229 |
+
do_sample=True,
|
| 230 |
+
temperature=0.2,
|
| 231 |
+
top_p=0.9,
|
| 232 |
+
top_k=20,
|
| 233 |
+
pad_token_id=tokenizer.eos_token_id,
|
| 234 |
+
repetition_penalty=1.0,
|
| 235 |
+
)
|
| 236 |
+
|
| 237 |
+
if request.stream:
|
| 238 |
+
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
|
| 239 |
+
gen_kwargs["streamer"] = streamer
|
| 240 |
+
thread = threading.Thread(target=run_generation, args=(model, tokenizer, gen_kwargs))
|
| 241 |
+
thread.start()
|
| 242 |
+
|
| 243 |
+
async def stream_generator():
|
| 244 |
+
full_text = ""
|
| 245 |
+
for text in streamer:
|
| 246 |
+
if text:
|
| 247 |
+
full_text += text
|
| 248 |
+
yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]}, ensure_ascii=False)}\n\n"
|
| 249 |
+
save_message("assistant", full_text)
|
| 250 |
+
yield "data: [DONE]\n\n"
|
| 251 |
+
|
| 252 |
+
return StreamingResponse(stream_generator(), media_type="text/event-stream; charset=utf-8")
|
| 253 |
+
|
| 254 |
+
else:
|
| 255 |
+
outputs = model.generate(**gen_kwargs)
|
| 256 |
+
resp_text = tokenizer.decode(outputs[0][prompt_ids.shape[1]:], skip_special_tokens=True)
|
| 257 |
+
save_message("assistant", resp_text)
|
| 258 |
+
return {"choices": [{"message": {"role": "assistant", "content": resp_text}, "finish_reason": "stop"}]}
|
| 259 |
+
|
| 260 |
+
if __name__ == "__main__":
|
| 261 |
+
uvicorn.run("server:app", host=HOST, port=PORT, reload=False)
|