ItamarMordo commited on
Commit
131c535
·
verified ·
1 Parent(s): dbb2788

Update server.py

Browse files
Files changed (1) hide show
  1. server.py +265 -260
server.py CHANGED
@@ -1,261 +1,266 @@
1
- import uvicorn
2
- import torch
3
- import threading
4
- import json
5
- import logging
6
- import sqlite3
7
- import os
8
- import re
9
- import time
10
- from typing import List, Optional, Literal, Tuple
11
- from fastapi import FastAPI, HTTPException
12
- from fastapi.middleware.cors import CORSMiddleware
13
- from fastapi.responses import StreamingResponse
14
- from pydantic import BaseModel
15
- from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
16
-
17
- # --- Configuration ---
18
- # HF Spaces uses port 7860 by default
19
- PORT = int(os.environ.get("PORT", 7860))
20
- HOST = "0.0.0.0"
21
-
22
- MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
23
- DB_FILE = "chatbot.db"
24
- JSON_SOURCE = "database.json"
25
-
26
- # --- Logging ---
27
- logging.basicConfig(level=logging.INFO)
28
- logger = logging.getLogger(__name__)
29
-
30
- # --- FastAPI ---
31
- app = FastAPI(title="Hebrew RAG Chatbot", version="4.0.0")
32
-
33
- app.add_middleware(
34
- CORSMiddleware,
35
- allow_origins=["*"],
36
- allow_credentials=True,
37
- allow_methods=["*"],
38
- allow_headers=["*"],
39
- )
40
-
41
- # --- Globals ---
42
- model = None
43
- tokenizer = None
44
- device = "cpu"
45
-
46
- # --- Hebrew Helpers ---
47
- STOP_WORDS = {
48
- "של", "את", "על", "עם", "כי", "זה", "או", "אם", "גם", "הוא",
49
- "היא", "הם", "מה", "מי", "איך", "למה", "כל", "לא", "כן", "אני",
50
- "אתה", "אנחנו", "הזה", "הזו", "שלי", "שלך", "בין", "רק", "עוד",
51
- "תסביר", "לי", "ספר", "בבקשה", "היי", "שלום", "תודה", "כיצד"
52
- }
53
-
54
- SYNONYMS = {
55
- "מתקינים": ["התקנה", "להתקין", "התקנת", "install"],
56
- "התקנה": ["מתקינים", "להתקין", "setup", "installation"],
57
- "מחברים": ["חיבור", "לחבר", "חיווט", "connect"],
58
- "חיבור": ["מחברים", "לחבר", "כבל", "wiring"],
59
- "תקלה": ["בעיה", "לא עובד", "שגיאה", "error", "bug"],
60
- "קוד": ["תכנות", "פרוייקט", "קבצים", "code"],
61
- "esp32": ["בקר", "מיקרובקר", "צ'יפ", "controller"],
62
- "nextion": ["מסך", "תצוגה", "HMI", "screen"],
63
- "מערכת": ["פרויקט", "בית חכם", "מוצר", "system"],
64
- "אפליקציה": ["יישום", "app", "apk"],
65
- "אימון": ["למידה", "train", "training", "ai"]
66
- }
67
-
68
- def clean_and_expand_query(text: str) -> List[str]:
69
- text = re.sub(r'[^\w\s]', ' ', text)
70
- words = text.split()
71
- search_terms = set()
72
- for w in words:
73
- if w in STOP_WORDS or len(w) < 2: continue
74
- search_terms.add(w)
75
- if w in SYNONYMS:
76
- for synonym in SYNONYMS[w]: search_terms.add(synonym)
77
- if len(w) > 3 and w[0] in ['ה', 'ב', 'ל', 'מ']:
78
- root = w[1:]
79
- search_terms.add(root)
80
- if root in SYNONYMS:
81
- for synonym in SYNONYMS[root]: search_terms.add(synonym)
82
- return list(search_terms)
83
-
84
- # --- Database Logic ---
85
- def init_db():
86
- if os.path.exists(DB_FILE): os.remove(DB_FILE)
87
- conn = sqlite3.connect(DB_FILE)
88
- cursor = conn.cursor()
89
- cursor.execute('CREATE TABLE IF NOT EXISTS knowledge (id INTEGER PRIMARY KEY, topic TEXT, content TEXT)')
90
- cursor.execute('CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY, role TEXT, content TEXT)')
91
-
92
- if os.path.exists(JSON_SOURCE):
93
- try:
94
- with open(JSON_SOURCE, 'r', encoding='utf-8') as f:
95
- data = json.load(f)
96
- for item in data:
97
- if isinstance(item, dict) and 'content' in item:
98
- cursor.execute('INSERT INTO knowledge (topic, content) VALUES (?, ?)',
99
- (item.get('topic', 'General'), item['content']))
100
- conn.commit()
101
- logger.info(f"✅ SUCCESS: Loaded articles from {JSON_SOURCE}")
102
- except Exception as e:
103
- logger.error(f"❌ ERROR reading JSON: {e}")
104
- conn.close()
105
-
106
- def query_knowledge_base(query: str) -> List[Tuple[str, str]]:
107
- conn = sqlite3.connect(DB_FILE)
108
- cursor = conn.cursor()
109
- search_terms = clean_and_expand_query(query)
110
- results = []
111
- seen_ids = set()
112
- for word in search_terms:
113
- cursor.execute("SELECT id, topic, content FROM knowledge WHERE content LIKE ?", (f'%{word}%',))
114
- for row_id, topic, content in cursor.fetchall():
115
- if row_id not in seen_ids:
116
- results.append((topic, content))
117
- seen_ids.add(row_id)
118
- conn.close()
119
- return results[:3]
120
-
121
- def save_message(role: str, content: str):
122
- try:
123
- conn = sqlite3.connect(DB_FILE)
124
- cursor = conn.cursor()
125
- cursor.execute("INSERT INTO history (role, content) VALUES (?, ?)", (role, content))
126
- conn.commit()
127
- conn.close()
128
- except Exception as e:
129
- logger.error(f"DB Write Error: {e}")
130
-
131
- # --- Models ---
132
- class Message(BaseModel):
133
- role: Literal["user", "assistant", "system"]
134
- content: str
135
-
136
- class ChatCompletionRequest(BaseModel):
137
- messages: List[Message]
138
- max_tokens: Optional[int] = 512
139
- temperature: Optional[float] = 0.2
140
- top_p: Optional[float] = 0.9
141
- stream: Optional[bool] = False
142
-
143
- # --- Startup ---
144
- @app.on_event("startup")
145
- async def startup_event():
146
- global model, tokenizer, device
147
- init_db()
148
-
149
- logger.info(f"Loading model: {MODEL_ID}...")
150
- device = "cuda" if torch.cuda.is_available() else "cpu"
151
- logger.info(f"Using device: {device}")
152
-
153
- try:
154
- tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
155
- model = AutoModelForCausalLM.from_pretrained(
156
- MODEL_ID,
157
- torch_dtype=torch.float16 if device != "cpu" else torch.float32,
158
- device_map=device,
159
- low_cpu_mem_usage=True,
160
- trust_remote_code=True
161
- )
162
- logger.info("Model loaded successfully!")
163
-
164
- logger.info("Performing warmup...")
165
- dummy_input = tokenizer("Hello", return_tensors="pt").to(device)
166
- model.generate(**dummy_input, max_new_tokens=1)
167
- logger.info("Warmup complete!")
168
-
169
- except Exception as e:
170
- logger.error(f"Failed to load: {e}")
171
- raise RuntimeError("Model load failed")
172
-
173
- def run_generation(model, tokenizer, params):
174
- try:
175
- model.generate(**params)
176
- except Exception as e:
177
- logger.error(f"Generation error: {e}")
178
-
179
- @app.get("/health")
180
- def health_check():
181
- return {"status": "running", "model": MODEL_ID, "device": device}
182
-
183
- @app.get("/")
184
- def home():
185
- return {"message": "Chatbot Server is Running. Use POST /v1/chat/completions"}
186
-
187
- @app.post("/v1/chat/completions")
188
- async def chat_completions(request: ChatCompletionRequest):
189
- if not model or not tokenizer:
190
- raise HTTPException(status_code=503, detail="Model not loaded")
191
-
192
- last_msg = next((m for m in reversed(request.messages) if m.role == "user"), None)
193
- if last_msg:
194
- save_message("user", last_msg.content)
195
- retrieved = query_knowledge_base(last_msg.content)
196
-
197
- sys_prompt = (
198
- "System: You are a helpful assistant for the 'Smart Home' project. "
199
- "Instructions:\n"
200
- "1. Answer ONLY in Hebrew (עברית).\n"
201
- "2. Use the provided Context to answer the question.\n"
202
- "3. If the answer is not in the context, say politely in Hebrew that you don't have the info.\n"
203
- "4. Do NOT start sentences with English words."
204
- )
205
-
206
- if retrieved:
207
- topics = ", ".join(sorted(list(set([i[0] for i in retrieved]))))
208
- content = "\n---\n".join([i[1] for i in retrieved])
209
- sys_prompt += f"\n\nContext ({topics}):\n{content}\n\nInstruction: Answer in Hebrew based on the context above."
210
- else:
211
- sys_prompt += "\n\nContext: No specific document found. Instruction: If this is a greeting (Shalom), reply politely in Hebrew. Otherwise, say you don't know."
212
-
213
- if request.messages[0].role == "system":
214
- request.messages[0].content = sys_prompt
215
- else:
216
- request.messages.insert(0, Message(role="system", content=sys_prompt))
217
-
218
- try:
219
- conversation = [msg.dict() for msg in request.messages]
220
- prompt_ids = tokenizer.apply_chat_template(conversation, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(device)
221
- attention_mask = torch.ones_like(prompt_ids)
222
- except Exception as e:
223
- raise HTTPException(status_code=400, detail=str(e))
224
-
225
- gen_kwargs = dict(
226
- inputs=prompt_ids,
227
- attention_mask=attention_mask,
228
- max_new_tokens=request.max_tokens,
229
- do_sample=True,
230
- temperature=0.2,
231
- top_p=0.9,
232
- top_k=20,
233
- pad_token_id=tokenizer.eos_token_id,
234
- repetition_penalty=1.0,
235
- )
236
-
237
- if request.stream:
238
- streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
239
- gen_kwargs["streamer"] = streamer
240
- thread = threading.Thread(target=run_generation, args=(model, tokenizer, gen_kwargs))
241
- thread.start()
242
-
243
- async def stream_generator():
244
- full_text = ""
245
- for text in streamer:
246
- if text:
247
- full_text += text
248
- yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]}, ensure_ascii=False)}\n\n"
249
- save_message("assistant", full_text)
250
- yield "data: [DONE]\n\n"
251
-
252
- return StreamingResponse(stream_generator(), media_type="text/event-stream; charset=utf-8")
253
-
254
- else:
255
- outputs = model.generate(**gen_kwargs)
256
- resp_text = tokenizer.decode(outputs[0][prompt_ids.shape[1]:], skip_special_tokens=True)
257
- save_message("assistant", resp_text)
258
- return {"choices": [{"message": {"role": "assistant", "content": resp_text}, "finish_reason": "stop"}]}
259
-
260
- if __name__ == "__main__":
 
 
 
 
 
261
  uvicorn.run("server:app", host=HOST, port=PORT, reload=False)
 
1
+ import uvicorn
2
+ import torch
3
+ import threading
4
+ import json
5
+ import logging
6
+ import sqlite3
7
+ import os
8
+ import re
9
+ import time
10
+ from typing import List, Optional, Literal, Tuple
11
+ from fastapi import FastAPI, HTTPException
12
+ from fastapi.middleware.cors import CORSMiddleware
13
+ from fastapi.responses import StreamingResponse
14
+ from pydantic import BaseModel
15
+ from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
16
+
17
+ # --- Configuration ---
18
+ # שימוש במודל 1.5B למהירות מקסימלית על CPU
19
+ MODEL_ID = "Qwen/Qwen2.5-1.5B-Instruct"
20
+ PORT = int(os.environ.get("PORT", 8000))
21
+ HOST = "0.0.0.0"
22
+ DB_FILE = "chatbot.db"
23
+ JSON_SOURCE = "database.json"
24
+
25
+ # --- Logging ---
26
+ logging.basicConfig(level=logging.INFO)
27
+ logger = logging.getLogger(__name__)
28
+
29
+ # --- FastAPI ---
30
+ app = FastAPI(title="Hebrew RAG Chatbot (Coherent)", version="3.8.0")
31
+
32
+ app.add_middleware(
33
+ CORSMiddleware,
34
+ allow_origins=["*"],
35
+ allow_credentials=True,
36
+ allow_methods=["*"],
37
+ allow_headers=["*"],
38
+ )
39
+
40
+ # --- Globals ---
41
+ model = None
42
+ tokenizer = None
43
+ device = "cpu"
44
+
45
+ # --- Hebrew Helpers ---
46
+ STOP_WORDS = {
47
+ "של", "את", "על", "עם", "כי", "זה", "או", "אם", "גם", "הוא",
48
+ "היא", "הם", "מה", "מי", "איך", "למה", "כל", "לא", "כן", "אני",
49
+ "אתה", "אנחנו", "הזה", "הזו", "שלי", "שלך", "בין", "רק", "עוד",
50
+ "תסביר", "לי", "ספר", "בבקשה", "היי", "שלום", "תודה", "כיצד"
51
+ }
52
+
53
+ SYNONYMS = {
54
+ "מתקינים": ["התקנה", "להתקין", "התקנת", "install"],
55
+ "התקנה": ["מתקינים", "להתקין", "setup", "installation"],
56
+ "מחברים": ["חיבור", "לחבר", "חיווט", "connect"],
57
+ "חיבור": ["מחברים", "לחבר", "כבל", "wiring"],
58
+ "תקלה": ["בעיה", "לא עובד", "שגיאה", "error", "bug"],
59
+ "קוד": ["תכנות", "פרוייקט", "קבצים", "code"],
60
+ "esp32": ["בקר", "מיקרובקר", "צ'יפ", "controller"],
61
+ "nextion": ["מסך", "תצוגה", "HMI", "screen"],
62
+ "מערכת": ["פרויקט", "בית חכם", "מוצר", "system"],
63
+ "אפליקציה": ["יישום", "app", "apk"],
64
+ "אימון": ["למידה", "train", "training", "ai"]
65
+ }
66
+
67
+ def clean_and_expand_query(text: str) -> List[str]:
68
+ text = re.sub(r'[^\w\s]', ' ', text)
69
+ words = text.split()
70
+ search_terms = set()
71
+ for w in words:
72
+ if w in STOP_WORDS or len(w) < 2: continue
73
+ search_terms.add(w)
74
+ if w in SYNONYMS:
75
+ for synonym in SYNONYMS[w]: search_terms.add(synonym)
76
+ if len(w) > 3 and w[0] in ['ה', 'ב', 'ל', 'מ']:
77
+ root = w[1:]
78
+ search_terms.add(root)
79
+ if root in SYNONYMS:
80
+ for synonym in SYNONYMS[root]: search_terms.add(synonym)
81
+ return list(search_terms)
82
+
83
+ # --- Database Logic ---
84
+ def init_db():
85
+ if os.path.exists(DB_FILE): os.remove(DB_FILE)
86
+ conn = sqlite3.connect(DB_FILE)
87
+ cursor = conn.cursor()
88
+ cursor.execute('CREATE TABLE IF NOT EXISTS knowledge (id INTEGER PRIMARY KEY, topic TEXT, content TEXT)')
89
+ cursor.execute('CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY, role TEXT, content TEXT)')
90
+
91
+ if os.path.exists(JSON_SOURCE):
92
+ try:
93
+ with open(JSON_SOURCE, 'r', encoding='utf-8') as f:
94
+ data = json.load(f)
95
+ for item in data:
96
+ if isinstance(item, dict) and 'content' in item:
97
+ cursor.execute('INSERT INTO knowledge (topic, content) VALUES (?, ?)',
98
+ (item.get('topic', 'General'), item['content']))
99
+ conn.commit()
100
+ logger.info(f"✅ SUCCESS: Loaded articles from {JSON_SOURCE}")
101
+ except Exception as e:
102
+ logger.error(f"❌ ERROR reading JSON: {e}")
103
+ conn.close()
104
+
105
+ def query_knowledge_base(query: str) -> List[Tuple[str, str]]:
106
+ conn = sqlite3.connect(DB_FILE)
107
+ cursor = conn.cursor()
108
+ search_terms = clean_and_expand_query(query)
109
+ results = []
110
+ seen_ids = set()
111
+ for word in search_terms:
112
+ cursor.execute("SELECT id, topic, content FROM knowledge WHERE content LIKE ?", (f'%{word}%',))
113
+ for row_id, topic, content in cursor.fetchall():
114
+ if row_id not in seen_ids:
115
+ results.append((topic, content))
116
+ seen_ids.add(row_id)
117
+ conn.close()
118
+ return results[:3]
119
+
120
+ def save_message(role: str, content: str):
121
+ try:
122
+ conn = sqlite3.connect(DB_FILE)
123
+ cursor = conn.cursor()
124
+ cursor.execute("INSERT INTO history (role, content) VALUES (?, ?)", (role, content))
125
+ conn.commit()
126
+ conn.close()
127
+ except Exception as e:
128
+ logger.error(f"DB Write Error: {e}")
129
+
130
+ # --- Models ---
131
+ class Message(BaseModel):
132
+ role: Literal["user", "assistant", "system"]
133
+ content: str
134
+
135
+ class ChatCompletionRequest(BaseModel):
136
+ messages: List[Message]
137
+ max_tokens: Optional[int] = 512
138
+ temperature: Optional[float] = 0.3 # טמפרטורה מאוזנת לזרימה טובה
139
+ top_p: Optional[float] = 0.9
140
+ stream: Optional[bool] = False
141
+
142
+ # --- Startup ---
143
+ @app.on_event("startup")
144
+ async def startup_event():
145
+ global model, tokenizer, device
146
+ init_db()
147
+
148
+ logger.info(f"Loading model: {MODEL_ID}...")
149
+ device = "cuda" if torch.cuda.is_available() else "cpu"
150
+ logger.info(f"Using device: {device}")
151
+
152
+ try:
153
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
154
+ model = AutoModelForCausalLM.from_pretrained(
155
+ MODEL_ID,
156
+ torch_dtype=torch.float16 if device != "cpu" else torch.float32,
157
+ device_map=device,
158
+ low_cpu_mem_usage=True,
159
+ trust_remote_code=True
160
+ )
161
+ logger.info("Model loaded successfully!")
162
+
163
+ logger.info("Performing warmup...")
164
+ dummy_input = tokenizer("Hello", return_tensors="pt").to(device)
165
+ model.generate(**dummy_input, max_new_tokens=1)
166
+ logger.info("Warmup complete!")
167
+
168
+ except Exception as e:
169
+ logger.error(f"Failed to load: {e}")
170
+ raise RuntimeError("Model load failed")
171
+
172
+ def run_generation(model, tokenizer, params):
173
+ try:
174
+ model.generate(**params)
175
+ except Exception as e:
176
+ logger.error(f"Generation error: {e}")
177
+
178
+ @app.get("/health")
179
+ def health_check():
180
+ return {"status": "running", "model": MODEL_ID, "device": device}
181
+
182
+ @app.get("/")
183
+ def home():
184
+ return {"message": "Chatbot Server is Running. Use POST /v1/chat/completions"}
185
+
186
+ @app.post("/v1/chat/completions")
187
+ async def chat_completions(request: ChatCompletionRequest):
188
+ if not model or not tokenizer:
189
+ raise HTTPException(status_code=503, detail="Model not loaded")
190
+
191
+ last_msg = next((m for m in reversed(request.messages) if m.role == "user"), None)
192
+ if last_msg:
193
+ save_message("user", last_msg.content)
194
+ retrieved = query_knowledge_base(last_msg.content)
195
+
196
+ # הנחיית מערכת משופרת לטבעיות
197
+ sys_prompt = (
198
+ "System: You are a helpful assistant for the 'Telem Smart Home' project. "
199
+ "Instructions:\n"
200
+ "1. Answer ONLY in natural, fluent Hebrew (עברית).\n"
201
+ "2. Do NOT use English connectors like 'moreover', 'however', 'accordingly'. Use Hebrew equivalents like 'כמו כן', 'בנוסף', 'בהתאם'.\n"
202
+ "3. Base your answer strictly on the provided Context.\n"
203
+ "4. Be concise and friendly."
204
+ )
205
+
206
+ if retrieved:
207
+ topics = ", ".join(sorted(list(set([i[0] for i in retrieved]))))
208
+ content = "\n---\n".join([i[1] for i in retrieved])
209
+ sys_prompt += f"\n\nContext ({topics}):\n{content}\n\nInstruction: Answer in Hebrew using the context above."
210
+ else:
211
+ sys_prompt += (
212
+ "\n\nContext: No specific document found.\n"
213
+ "Instruction: If the user says 'Shalom' or greets you, answer politely in Hebrew. "
214
+ "For technical questions without context, apologize in Hebrew that you don't have the information."
215
+ )
216
+
217
+ if request.messages[0].role == "system":
218
+ request.messages[0].content = sys_prompt
219
+ else:
220
+ request.messages.insert(0, Message(role="system", content=sys_prompt))
221
+
222
+ try:
223
+ conversation = [msg.dict() for msg in request.messages]
224
+ prompt_ids = tokenizer.apply_chat_template(conversation, tokenize=True, add_generation_prompt=True, return_tensors="pt").to(device)
225
+ attention_mask = torch.ones_like(prompt_ids)
226
+ except Exception as e:
227
+ raise HTTPException(status_code=400, detail=str(e))
228
+
229
+ gen_kwargs = dict(
230
+ inputs=prompt_ids,
231
+ attention_mask=attention_mask,
232
+ max_new_tokens=request.max_tokens,
233
+ do_sample=True,
234
+ temperature=0.3, # מעט גמישות לשפה טבעית
235
+ top_p=0.9,
236
+ top_k=40,
237
+ pad_token_id=tokenizer.eos_token_id,
238
+ repetition_penalty=1.1, # ענישה מתונה למניעת לופים
239
+ no_repeat_ngram_size=3 # מונע חזרה על משפטים שלמים (חובה נגד לופים!)
240
+ )
241
+
242
+ if request.stream:
243
+ streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
244
+ gen_kwargs["streamer"] = streamer
245
+ thread = threading.Thread(target=run_generation, args=(model, tokenizer, gen_kwargs))
246
+ thread.start()
247
+
248
+ async def stream_generator():
249
+ full_text = ""
250
+ for text in streamer:
251
+ if text:
252
+ full_text += text
253
+ yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]}, ensure_ascii=False)}\n\n"
254
+ save_message("assistant", full_text)
255
+ yield "data: [DONE]\n\n"
256
+
257
+ return StreamingResponse(stream_generator(), media_type="text/event-stream; charset=utf-8")
258
+
259
+ else:
260
+ outputs = model.generate(**gen_kwargs)
261
+ resp_text = tokenizer.decode(outputs[0][prompt_ids.shape[1]:], skip_special_tokens=True)
262
+ save_message("assistant", resp_text)
263
+ return {"choices": [{"message": {"role": "assistant", "content": resp_text}, "finish_reason": "stop"}]}
264
+
265
+ if __name__ == "__main__":
266
  uvicorn.run("server:app", host=HOST, port=PORT, reload=False)