ItamarMordo commited on
Commit
9ebed6d
·
verified ·
1 Parent(s): 3614b16

Update server.py

Browse files
Files changed (1) hide show
  1. server.py +16 -6
server.py CHANGED
@@ -23,7 +23,7 @@ JSON_SOURCE = "database.json"
23
  logging.basicConfig(level=logging.INFO)
24
  logger = logging.getLogger(__name__)
25
 
26
- app = FastAPI(title="Hebrew Smart Home Technician", version="9.2.0")
27
 
28
  app.add_middleware(
29
  CORSMiddleware,
@@ -117,6 +117,7 @@ def query_knowledge_base(query: str) -> Tuple[str, List[str]]:
117
  results = []
118
  seen_ids = set()
119
 
 
120
  for term in search_terms:
121
  cursor.execute("SELECT id, content, topic FROM knowledge WHERE content LIKE ? OR topic LIKE ?", (f'%{term}%', f'%{term}%'))
122
  rows = cursor.fetchall()
@@ -139,6 +140,13 @@ def query_knowledge_base(query: str) -> Tuple[str, List[str]]:
139
 
140
  results.append((score, content))
141
  seen_ids.add(row_id)
 
 
 
 
 
 
 
142
  conn.close()
143
 
144
  results.sort(key=lambda x: x[0], reverse=True)
@@ -166,7 +174,7 @@ class Message(BaseModel):
166
 
167
  class ChatCompletionRequest(BaseModel):
168
  messages: List[Message]
169
- max_tokens: Optional[int] = 512
170
  temperature: Optional[float] = 0.3
171
  stream: Optional[bool] = False
172
 
@@ -204,7 +212,8 @@ async def chat_completions(request: ChatCompletionRequest):
204
 
205
  context = ""
206
  if last_user_msg:
207
- save_message("user", last_user_msg.content)
 
208
  context, _ = query_knowledge_base(last_user_msg.content)
209
 
210
  if context:
@@ -232,12 +241,12 @@ async def chat_completions(request: ChatCompletionRequest):
232
 
233
  gen_kwargs = dict(
234
  inputs=prompt_ids,
235
- max_new_tokens=request.max_tokens,
236
  temperature=0.3,
237
  top_p=0.9,
238
  do_sample=True,
239
  pad_token_id=tokenizer.eos_token_id,
240
- repetition_penalty=1.1 # התיקון ללופים: ענישה עדינה
241
  )
242
 
243
  if request.stream:
@@ -252,7 +261,8 @@ async def chat_completions(request: ChatCompletionRequest):
252
  if text:
253
  full_text += text
254
  yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]}, ensure_ascii=False)}\n\n"
255
- save_message("assistant", full_text)
 
256
  yield "data: [DONE]\n\n"
257
 
258
  return StreamingResponse(stream_generator(), media_type="text/event-stream; charset=utf-8")
 
23
  logging.basicConfig(level=logging.INFO)
24
  logger = logging.getLogger(__name__)
25
 
26
+ app = FastAPI(title="Hebrew Smart Home Technician", version="9.3.0")
27
 
28
  app.add_middleware(
29
  CORSMiddleware,
 
117
  results = []
118
  seen_ids = set()
119
 
120
+ # Optimzation: Limit query to finding top match quickly
121
  for term in search_terms:
122
  cursor.execute("SELECT id, content, topic FROM knowledge WHERE content LIKE ? OR topic LIKE ?", (f'%{term}%', f'%{term}%'))
123
  rows = cursor.fetchall()
 
140
 
141
  results.append((score, content))
142
  seen_ids.add(row_id)
143
+
144
+ # OPTIMIZATION: Early exit if perfect match found
145
+ if score >= 100:
146
+ break
147
+ if results and results[-1][0] >= 100:
148
+ break
149
+
150
  conn.close()
151
 
152
  results.sort(key=lambda x: x[0], reverse=True)
 
174
 
175
  class ChatCompletionRequest(BaseModel):
176
  messages: List[Message]
177
+ max_tokens: Optional[int] = 256 # OPTIMIZATION: Reduced from 512 for speed
178
  temperature: Optional[float] = 0.3
179
  stream: Optional[bool] = False
180
 
 
212
 
213
  context = ""
214
  if last_user_msg:
215
+ # Don't await DB save to speed up response start (fire and forget logic for logging)
216
+ threading.Thread(target=save_message, args=("user", last_user_msg.content)).start()
217
  context, _ = query_knowledge_base(last_user_msg.content)
218
 
219
  if context:
 
241
 
242
  gen_kwargs = dict(
243
  inputs=prompt_ids,
244
+ max_new_tokens=request.max_tokens, # Now 256
245
  temperature=0.3,
246
  top_p=0.9,
247
  do_sample=True,
248
  pad_token_id=tokenizer.eos_token_id,
249
+ repetition_penalty=1.1
250
  )
251
 
252
  if request.stream:
 
261
  if text:
262
  full_text += text
263
  yield f"data: {json.dumps({'choices': [{'delta': {'content': text}}]}, ensure_ascii=False)}\n\n"
264
+ # Save assistant response in background
265
+ threading.Thread(target=save_message, args=("assistant", full_text)).start()
266
  yield "data: [DONE]\n\n"
267
 
268
  return StreamingResponse(stream_generator(), media_type="text/event-stream; charset=utf-8")