Spaces:
Sleeping
Sleeping
Update app/app.py
Browse files- app/app.py +106 -85
app/app.py
CHANGED
|
@@ -4,11 +4,28 @@ import asyncio
|
|
| 4 |
import logging
|
| 5 |
import uuid
|
| 6 |
import re
|
|
|
|
| 7 |
from fastapi import FastAPI, HTTPException, Request
|
| 8 |
from pydantic import BaseModel
|
| 9 |
from llama_cpp import Llama
|
| 10 |
from app.policy_vector_db import PolicyVectorDB, ensure_db_populated
|
| 11 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 12 |
# -----------------------------
|
| 13 |
# ✅ Logging Configuration
|
| 14 |
# -----------------------------
|
|
@@ -20,21 +37,10 @@ class RequestIdAdapter(logging.LoggerAdapter):
|
|
| 20 |
|
| 21 |
logger = logging.getLogger("app")
|
| 22 |
|
| 23 |
-
# -----------------------------
|
| 24 |
-
# ✅ Configuration
|
| 25 |
-
# -----------------------------
|
| 26 |
-
DB_PERSIST_DIRECTORY = os.getenv("DB_PERSIST_DIRECTORY", "/app/vector_database")
|
| 27 |
-
CHUNKS_FILE_PATH = os.getenv("CHUNKS_FILE_PATH", "/app/granular_chunks_final.jsonl")
|
| 28 |
-
MODEL_PATH = os.getenv("MODEL_PATH", "/app/tinyllama_dop_q4_k_m.gguf")
|
| 29 |
-
LLM_TIMEOUT_SECONDS = int(os.getenv("LLM_TIMEOUT_SECONDS", "90"))
|
| 30 |
-
RELEVANCE_THRESHOLD = float(os.getenv("RELEVANCE_THRESHOLD", "0.3"))
|
| 31 |
-
TOP_K_SEARCH = int(os.getenv("TOP_K_SEARCH", "5")) # Increased for better context
|
| 32 |
-
TOP_K_CONTEXT = int(os.getenv("TOP_K_CONTEXT", "3")) # Use more context chunks
|
| 33 |
-
|
| 34 |
# -----------------------------
|
| 35 |
# ✅ Initialize FastAPI App
|
| 36 |
# -----------------------------
|
| 37 |
-
app = FastAPI(title="NEEPCO DoP RAG Chatbot", version="2.
|
| 38 |
|
| 39 |
@app.middleware("http")
|
| 40 |
async def add_request_id(request: Request, call_next):
|
|
@@ -45,7 +51,7 @@ async def add_request_id(request: Request, call_next):
|
|
| 45 |
return response
|
| 46 |
|
| 47 |
# -----------------------------
|
| 48 |
-
# ✅ Vector DB
|
| 49 |
# -----------------------------
|
| 50 |
logger.info("Initializing vector DB...")
|
| 51 |
try:
|
|
@@ -66,25 +72,37 @@ except Exception as e:
|
|
| 66 |
db_ready = False
|
| 67 |
|
| 68 |
# -----------------------------
|
| 69 |
-
# ✅
|
| 70 |
# -----------------------------
|
| 71 |
-
logger.info(f"Loading GGUF model from: {MODEL_PATH}")
|
| 72 |
try:
|
| 73 |
llm = Llama(
|
| 74 |
model_path=MODEL_PATH,
|
| 75 |
-
n_ctx=
|
| 76 |
-
n_threads=
|
| 77 |
-
n_batch=
|
| 78 |
-
use_mlock=False,
|
| 79 |
-
|
|
|
|
|
|
|
|
|
|
| 80 |
)
|
| 81 |
-
logger.info("GGUF model loaded successfully.")
|
| 82 |
model_ready = True
|
| 83 |
except Exception as e:
|
| 84 |
logger.error(f"FATAL: Failed to load GGUF model: {e}", exc_info=True)
|
| 85 |
llm = None
|
| 86 |
model_ready = False
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
# -----------------------------
|
| 89 |
# ✅ API Schemas
|
| 90 |
# -----------------------------
|
|
@@ -100,25 +118,21 @@ class Feedback(BaseModel):
|
|
| 100 |
comment: str | None = None
|
| 101 |
|
| 102 |
# -----------------------------
|
| 103 |
-
# ✅ Enhanced Query Processing
|
| 104 |
# -----------------------------
|
| 105 |
def classify_query_type(question: str) -> str:
|
| 106 |
"""Classify the type of query to choose appropriate search strategy."""
|
| 107 |
question_lower = question.lower()
|
| 108 |
|
| 109 |
-
# Monetary queries
|
| 110 |
if re.search(r'₹|crore|lakh|\d+.*approve|limit.*\d+', question_lower):
|
| 111 |
return "monetary"
|
| 112 |
|
| 113 |
-
# Authority/Role queries
|
| 114 |
if any(word in question_lower for word in ["who can", "who approve", "authority", "delegation"]):
|
| 115 |
return "authority"
|
| 116 |
|
| 117 |
-
# Process/Procedure queries
|
| 118 |
if any(word in question_lower for word in ["how to", "procedure", "process", "steps", "requirement"]):
|
| 119 |
return "procedure"
|
| 120 |
|
| 121 |
-
# Specific section queries
|
| 122 |
if re.search(r'section|annexure|clause', question_lower):
|
| 123 |
return "section_specific"
|
| 124 |
|
|
@@ -134,8 +148,6 @@ def extract_monetary_amount(question: str) -> float:
|
|
| 134 |
r'₹\s*(\d+(?:,\d+)*(?:\.\d+)?)'
|
| 135 |
]
|
| 136 |
|
| 137 |
-
multipliers = {'crore': 1e7, 'lakh': 1e5}
|
| 138 |
-
|
| 139 |
for pattern in patterns:
|
| 140 |
match = re.search(pattern, question, re.IGNORECASE)
|
| 141 |
if match:
|
|
@@ -151,7 +163,6 @@ def extract_monetary_amount(question: str) -> float:
|
|
| 151 |
def build_enhanced_prompt(question: str, context: str, query_type: str, search_results: list) -> str:
|
| 152 |
"""Build context-aware prompt based on query type and metadata."""
|
| 153 |
|
| 154 |
-
# Extract metadata insights
|
| 155 |
roles_mentioned = set()
|
| 156 |
sections_mentioned = set()
|
| 157 |
|
|
@@ -162,7 +173,6 @@ def build_enhanced_prompt(question: str, context: str, query_type: str, search_r
|
|
| 162 |
if 'section' in metadata:
|
| 163 |
sections_mentioned.add(metadata['section'])
|
| 164 |
|
| 165 |
-
# Build context-aware instructions
|
| 166 |
type_instructions = {
|
| 167 |
"monetary": "Focus on monetary limits, delegation amounts, and approval authorities for the specified amount.",
|
| 168 |
"authority": "Clearly identify the specific roles/positions and their delegation limits.",
|
|
@@ -173,7 +183,6 @@ def build_enhanced_prompt(question: str, context: str, query_type: str, search_r
|
|
| 173 |
|
| 174 |
instruction = type_instructions.get(query_type, type_instructions["general"])
|
| 175 |
|
| 176 |
-
# Add metadata context if available
|
| 177 |
metadata_context = ""
|
| 178 |
if roles_mentioned:
|
| 179 |
metadata_context += f"\nRoles involved: {', '.join(roles_mentioned)}"
|
|
@@ -197,7 +206,6 @@ Your task is to answer the user's question based ONLY on the provided context.
|
|
| 197 |
</s>
|
| 198 |
<|user|>
|
| 199 |
### Relevant Policy Context:
|
| 200 |
-
|
| 201 |
{context}
|
| 202 |
|
| 203 |
### Question:
|
|
@@ -208,6 +216,48 @@ Your task is to answer the user's question based ONLY on the provided context.
|
|
| 208 |
"""
|
| 209 |
return prompt
|
| 210 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
# -----------------------------
|
| 212 |
# ✅ Endpoints
|
| 213 |
# -----------------------------
|
|
@@ -216,36 +266,35 @@ def get_logger_adapter(request: Request):
|
|
| 216 |
|
| 217 |
@app.get("/")
|
| 218 |
async def root():
|
| 219 |
-
return {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
|
| 221 |
@app.get("/health")
|
| 222 |
async def health_check():
|
| 223 |
status = {
|
| 224 |
"status": "ok",
|
| 225 |
"database_status": "ready" if db_ready else "error",
|
| 226 |
-
"model_status": "ready" if model_ready else "error"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 227 |
}
|
| 228 |
if not db_ready or not model_ready:
|
| 229 |
raise HTTPException(status_code=503, detail=status)
|
| 230 |
return status
|
| 231 |
|
| 232 |
-
async def generate_llm_response(prompt: str, request_id: str):
|
| 233 |
-
loop = asyncio.get_running_loop()
|
| 234 |
-
response = await loop.run_in_executor(
|
| 235 |
-
None,
|
| 236 |
-
lambda: llm(prompt, max_tokens=1028, stop=["###", "Question:", "Context:", "</s>"], temperature=0.2, echo=False)
|
| 237 |
-
)
|
| 238 |
-
answer = response["choices"][0]["text"].strip()
|
| 239 |
-
if not answer:
|
| 240 |
-
raise ValueError("Empty response from LLM")
|
| 241 |
-
return answer
|
| 242 |
-
|
| 243 |
@app.post("/chat")
|
| 244 |
async def chat(query: Query, request: Request):
|
| 245 |
adapter = get_logger_adapter(request)
|
| 246 |
question_lower = query.question.strip().lower()
|
| 247 |
|
| 248 |
-
#
|
| 249 |
greeting_keywords = ["hello", "hi", "hey", "what can you do", "who are you"]
|
| 250 |
if question_lower in greeting_keywords:
|
| 251 |
adapter.info(f"Handling a greeting or introductory query: '{query.question}'")
|
|
@@ -267,15 +316,13 @@ async def chat(query: Query, request: Request):
|
|
| 267 |
|
| 268 |
adapter.info(f"Received query: '{query.question}'")
|
| 269 |
|
| 270 |
-
#
|
| 271 |
query_type = classify_query_type(query.question)
|
| 272 |
adapter.info(f"Query classified as: {query_type}")
|
| 273 |
|
| 274 |
-
# ✅ NEW: Enhanced search strategy based on query type
|
| 275 |
search_results = []
|
| 276 |
|
| 277 |
if query_type == "monetary":
|
| 278 |
-
# Try specialized monetary search first
|
| 279 |
amount = extract_monetary_amount(query.question)
|
| 280 |
if amount:
|
| 281 |
adapter.info(f"Extracted monetary amount: ₹{amount}")
|
|
@@ -284,7 +331,6 @@ async def chat(query: Query, request: Request):
|
|
| 284 |
search_results = monetary_results
|
| 285 |
adapter.info(f"Found {len(search_results)} results using monetary search")
|
| 286 |
|
| 287 |
-
# Fallback to enhanced semantic search with context
|
| 288 |
if not search_results:
|
| 289 |
search_results = db.search_with_context(
|
| 290 |
query.question,
|
|
@@ -302,7 +348,7 @@ async def chat(query: Query, request: Request):
|
|
| 302 |
"answer": "Sorry, I could not find a relevant policy to answer that question. Please try rephrasing or ask about specific delegation limits, approval authorities, or procedures."
|
| 303 |
}
|
| 304 |
|
| 305 |
-
#
|
| 306 |
result_info = []
|
| 307 |
for i, result in enumerate(search_results):
|
| 308 |
metadata = result.get('metadata', {})
|
|
@@ -313,13 +359,12 @@ async def chat(query: Query, request: Request):
|
|
| 313 |
|
| 314 |
adapter.info(f"Search results: {' | '.join(result_info)}")
|
| 315 |
|
| 316 |
-
#
|
| 317 |
context_chunks = []
|
| 318 |
for result in search_results[:TOP_K_CONTEXT]:
|
| 319 |
chunk_text = result['text']
|
| 320 |
metadata = result.get('metadata', {})
|
| 321 |
|
| 322 |
-
# Add metadata context if available
|
| 323 |
if metadata.get('section') or metadata.get('role'):
|
| 324 |
metadata_prefix = f"[Section: {metadata.get('section', 'N/A')}, Role: {metadata.get('role', 'N/A')}] "
|
| 325 |
chunk_text = metadata_prefix + chunk_text
|
|
@@ -327,11 +372,9 @@ async def chat(query: Query, request: Request):
|
|
| 327 |
context_chunks.append(chunk_text)
|
| 328 |
|
| 329 |
context = "\n---\n".join(context_chunks)
|
| 330 |
-
|
| 331 |
-
# ✅ NEW: Build enhanced prompt
|
| 332 |
prompt = build_enhanced_prompt(query.question, context, query_type, search_results)
|
| 333 |
|
| 334 |
-
#
|
| 335 |
answer = "An error occurred while processing your request."
|
| 336 |
try:
|
| 337 |
adapter.info(f"Sending enhanced prompt to LLM for {query_type} query...")
|
|
@@ -341,17 +384,15 @@ async def chat(query: Query, request: Request):
|
|
| 341 |
)
|
| 342 |
adapter.info(f"LLM generation successful. Raw response: {raw_answer[:250]}...")
|
| 343 |
|
| 344 |
-
#
|
| 345 |
if '|' in raw_answer:
|
| 346 |
adapter.info("Pipe separator found. Formatting response as a bulleted list.")
|
| 347 |
items = raw_answer.split('|')
|
| 348 |
cleaned_items = [f"• {item.strip()}" for item in items if item.strip()]
|
| 349 |
answer = "\n".join(cleaned_items)
|
| 350 |
else:
|
| 351 |
-
# Clean up the answer
|
| 352 |
answer = raw_answer.strip()
|
| 353 |
|
| 354 |
-
# Add helpful formatting for different query types
|
| 355 |
if query_type == "monetary" and "₹" not in answer and extract_monetary_amount(query.question):
|
| 356 |
amount = extract_monetary_amount(query.question)
|
| 357 |
answer = f"For amounts of ₹{amount:,.0f}:\n\n{answer}"
|
|
@@ -369,7 +410,7 @@ async def chat(query: Query, request: Request):
|
|
| 369 |
"question": query.question,
|
| 370 |
"context_used": context,
|
| 371 |
"answer": answer,
|
| 372 |
-
"query_type": query_type,
|
| 373 |
"search_strategy": "monetary" if query_type == "monetary" and extract_monetary_amount(query.question) else "semantic_with_context"
|
| 374 |
}
|
| 375 |
|
|
@@ -388,28 +429,8 @@ async def collect_feedback(feedback: Feedback, request: Request):
|
|
| 388 |
adapter.info(json.dumps(feedback_log))
|
| 389 |
return {"status": "✅ Feedback recorded. Thank you!"}
|
| 390 |
|
| 391 |
-
# ✅
|
| 392 |
-
@app.
|
| 393 |
-
async def
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
raise HTTPException(status_code=503, detail="Database not ready")
|
| 397 |
-
|
| 398 |
-
adapter = get_logger_adapter(request)
|
| 399 |
-
query_type = classify_query_type(query.question)
|
| 400 |
-
|
| 401 |
-
# Get search results
|
| 402 |
-
search_results = db.search_with_context(query.question, top_k=10, include_related=True)
|
| 403 |
-
|
| 404 |
-
return {
|
| 405 |
-
"request_id": request.state.request_id,
|
| 406 |
-
"question": query.question,
|
| 407 |
-
"query_type": query_type,
|
| 408 |
-
"results_count": len(search_results),
|
| 409 |
-
"results": [{
|
| 410 |
-
"text": result['text'][:200] + "...",
|
| 411 |
-
"metadata": result.get('metadata', {}),
|
| 412 |
-
"relevance_score": result.get('relevance_score'),
|
| 413 |
-
"is_context": result.get('is_context', False)
|
| 414 |
-
} for result in search_results]
|
| 415 |
-
}
|
|
|
|
| 4 |
import logging
|
| 5 |
import uuid
|
| 6 |
import re
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
from fastapi import FastAPI, HTTPException, Request
|
| 9 |
from pydantic import BaseModel
|
| 10 |
from llama_cpp import Llama
|
| 11 |
from app.policy_vector_db import PolicyVectorDB, ensure_db_populated
|
| 12 |
|
| 13 |
+
# -----------------------------
|
| 14 |
+
# ✅ Enhanced Configuration for Maximum CPU Usage
|
| 15 |
+
# -----------------------------
|
| 16 |
+
DB_PERSIST_DIRECTORY = os.getenv("DB_PERSIST_DIRECTORY", "/app/vector_database")
|
| 17 |
+
CHUNKS_FILE_PATH = os.getenv("CHUNKS_FILE_PATH", "/app/granular_chunks_final.jsonl")
|
| 18 |
+
MODEL_PATH = os.getenv("MODEL_PATH", "/app/tinyllama_dop_q4_k_m.gguf")
|
| 19 |
+
LLM_TIMEOUT_SECONDS = int(os.getenv("LLM_TIMEOUT_SECONDS", "120")) # Increased timeout
|
| 20 |
+
RELEVANCE_THRESHOLD = float(os.getenv("RELEVANCE_THRESHOLD", "0.3"))
|
| 21 |
+
TOP_K_SEARCH = int(os.getenv("TOP_K_SEARCH", "5"))
|
| 22 |
+
TOP_K_CONTEXT = int(os.getenv("TOP_K_CONTEXT", "3"))
|
| 23 |
+
|
| 24 |
+
# ✅ NEW: CPU optimization parameters
|
| 25 |
+
CPU_CORES = os.cpu_count() or 4 # Detect available CPU cores
|
| 26 |
+
LLM_THREADS = min(CPU_CORES, 8) # Use all cores for LLM (max 8 for efficiency)
|
| 27 |
+
EXECUTOR_WORKERS = CPU_CORES * 2 # More workers for concurrent requests
|
| 28 |
+
|
| 29 |
# -----------------------------
|
| 30 |
# ✅ Logging Configuration
|
| 31 |
# -----------------------------
|
|
|
|
| 37 |
|
| 38 |
logger = logging.getLogger("app")
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
# -----------------------------
|
| 41 |
# ✅ Initialize FastAPI App
|
| 42 |
# -----------------------------
|
| 43 |
+
app = FastAPI(title="NEEPCO DoP RAG Chatbot", version="2.3.0")
|
| 44 |
|
| 45 |
@app.middleware("http")
|
| 46 |
async def add_request_id(request: Request, call_next):
|
|
|
|
| 51 |
return response
|
| 52 |
|
| 53 |
# -----------------------------
|
| 54 |
+
# ✅ Vector DB Initialization
|
| 55 |
# -----------------------------
|
| 56 |
logger.info("Initializing vector DB...")
|
| 57 |
try:
|
|
|
|
| 72 |
db_ready = False
|
| 73 |
|
| 74 |
# -----------------------------
|
| 75 |
+
# ✅ Optimized GGUF Model Loading
|
| 76 |
# -----------------------------
|
| 77 |
+
logger.info(f"Loading GGUF model with {LLM_THREADS} threads from: {MODEL_PATH}")
|
| 78 |
try:
|
| 79 |
llm = Llama(
|
| 80 |
model_path=MODEL_PATH,
|
| 81 |
+
n_ctx=4096, # Increased context size
|
| 82 |
+
n_threads=LLM_THREADS, # ✅ Use all available CPU cores
|
| 83 |
+
n_batch=1024, # Increased batch size for better throughput
|
| 84 |
+
use_mlock=False, # Disable memory locking for flexibility
|
| 85 |
+
use_mmap=True, # Enable memory mapping for efficiency
|
| 86 |
+
verbose=False,
|
| 87 |
+
n_gpu_layers=0, # CPU only
|
| 88 |
+
f16_kv=True, # Use 16-bit for key-value cache to save memory
|
| 89 |
)
|
| 90 |
+
logger.info(f"GGUF model loaded successfully with {LLM_THREADS} threads.")
|
| 91 |
model_ready = True
|
| 92 |
except Exception as e:
|
| 93 |
logger.error(f"FATAL: Failed to load GGUF model: {e}", exc_info=True)
|
| 94 |
llm = None
|
| 95 |
model_ready = False
|
| 96 |
|
| 97 |
+
# -----------------------------
|
| 98 |
+
# ✅ Dedicated Thread Pool for LLM Inference
|
| 99 |
+
# -----------------------------
|
| 100 |
+
llm_executor = ThreadPoolExecutor(
|
| 101 |
+
max_workers=EXECUTOR_WORKERS,
|
| 102 |
+
thread_name_prefix="LLM-Worker"
|
| 103 |
+
)
|
| 104 |
+
logger.info(f"Created LLM thread pool with {EXECUTOR_WORKERS} workers")
|
| 105 |
+
|
| 106 |
# -----------------------------
|
| 107 |
# ✅ API Schemas
|
| 108 |
# -----------------------------
|
|
|
|
| 118 |
comment: str | None = None
|
| 119 |
|
| 120 |
# -----------------------------
|
| 121 |
+
# ✅ Enhanced Query Processing Functions
|
| 122 |
# -----------------------------
|
| 123 |
def classify_query_type(question: str) -> str:
|
| 124 |
"""Classify the type of query to choose appropriate search strategy."""
|
| 125 |
question_lower = question.lower()
|
| 126 |
|
|
|
|
| 127 |
if re.search(r'₹|crore|lakh|\d+.*approve|limit.*\d+', question_lower):
|
| 128 |
return "monetary"
|
| 129 |
|
|
|
|
| 130 |
if any(word in question_lower for word in ["who can", "who approve", "authority", "delegation"]):
|
| 131 |
return "authority"
|
| 132 |
|
|
|
|
| 133 |
if any(word in question_lower for word in ["how to", "procedure", "process", "steps", "requirement"]):
|
| 134 |
return "procedure"
|
| 135 |
|
|
|
|
| 136 |
if re.search(r'section|annexure|clause', question_lower):
|
| 137 |
return "section_specific"
|
| 138 |
|
|
|
|
| 148 |
r'₹\s*(\d+(?:,\d+)*(?:\.\d+)?)'
|
| 149 |
]
|
| 150 |
|
|
|
|
|
|
|
| 151 |
for pattern in patterns:
|
| 152 |
match = re.search(pattern, question, re.IGNORECASE)
|
| 153 |
if match:
|
|
|
|
| 163 |
def build_enhanced_prompt(question: str, context: str, query_type: str, search_results: list) -> str:
|
| 164 |
"""Build context-aware prompt based on query type and metadata."""
|
| 165 |
|
|
|
|
| 166 |
roles_mentioned = set()
|
| 167 |
sections_mentioned = set()
|
| 168 |
|
|
|
|
| 173 |
if 'section' in metadata:
|
| 174 |
sections_mentioned.add(metadata['section'])
|
| 175 |
|
|
|
|
| 176 |
type_instructions = {
|
| 177 |
"monetary": "Focus on monetary limits, delegation amounts, and approval authorities for the specified amount.",
|
| 178 |
"authority": "Clearly identify the specific roles/positions and their delegation limits.",
|
|
|
|
| 183 |
|
| 184 |
instruction = type_instructions.get(query_type, type_instructions["general"])
|
| 185 |
|
|
|
|
| 186 |
metadata_context = ""
|
| 187 |
if roles_mentioned:
|
| 188 |
metadata_context += f"\nRoles involved: {', '.join(roles_mentioned)}"
|
|
|
|
| 206 |
</s>
|
| 207 |
<|user|>
|
| 208 |
### Relevant Policy Context:
|
|
|
|
| 209 |
{context}
|
| 210 |
|
| 211 |
### Question:
|
|
|
|
| 216 |
"""
|
| 217 |
return prompt
|
| 218 |
|
| 219 |
+
# -----------------------------
|
| 220 |
+
# ✅ Optimized LLM Response Generation
|
| 221 |
+
# -----------------------------
|
| 222 |
+
def generate_llm_response_sync(prompt: str, request_id: str) -> str:
|
| 223 |
+
"""Synchronous LLM generation for thread pool execution."""
|
| 224 |
+
try:
|
| 225 |
+
# ✅ Optimized parameters for better CPU utilization
|
| 226 |
+
response = llm(
|
| 227 |
+
prompt,
|
| 228 |
+
max_tokens=1536, # Increased token limit
|
| 229 |
+
stop=["###", "Question:", "Context:", "</s>"],
|
| 230 |
+
temperature=0.1, # Lower temperature for more consistent responses
|
| 231 |
+
top_p=0.9,
|
| 232 |
+
repeat_penalty=1.1,
|
| 233 |
+
echo=False
|
| 234 |
+
)
|
| 235 |
+
|
| 236 |
+
if response and "choices" in response and len(response["choices"]) > 0:
|
| 237 |
+
return response["choices"][0]["text"].strip()
|
| 238 |
+
else:
|
| 239 |
+
raise ValueError("Empty or invalid response from LLM")
|
| 240 |
+
|
| 241 |
+
except Exception as e:
|
| 242 |
+
logger.error(f"LLM generation error for request {request_id}: {e}")
|
| 243 |
+
raise
|
| 244 |
+
|
| 245 |
+
async def generate_llm_response(prompt: str, request_id: str) -> str:
|
| 246 |
+
"""Async wrapper for LLM generation using dedicated thread pool."""
|
| 247 |
+
loop = asyncio.get_running_loop()
|
| 248 |
+
try:
|
| 249 |
+
# ✅ Use dedicated thread pool for better CPU utilization
|
| 250 |
+
response = await loop.run_in_executor(
|
| 251 |
+
llm_executor,
|
| 252 |
+
generate_llm_response_sync,
|
| 253 |
+
prompt,
|
| 254 |
+
request_id
|
| 255 |
+
)
|
| 256 |
+
return response
|
| 257 |
+
except Exception as e:
|
| 258 |
+
logger.error(f"Async LLM generation error: {e}")
|
| 259 |
+
raise
|
| 260 |
+
|
| 261 |
# -----------------------------
|
| 262 |
# ✅ Endpoints
|
| 263 |
# -----------------------------
|
|
|
|
| 266 |
|
| 267 |
@app.get("/")
|
| 268 |
async def root():
|
| 269 |
+
return {
|
| 270 |
+
"status": "✅ Server is running.",
|
| 271 |
+
"cpu_cores": CPU_CORES,
|
| 272 |
+
"llm_threads": LLM_THREADS,
|
| 273 |
+
"executor_workers": EXECUTOR_WORKERS
|
| 274 |
+
}
|
| 275 |
|
| 276 |
@app.get("/health")
|
| 277 |
async def health_check():
|
| 278 |
status = {
|
| 279 |
"status": "ok",
|
| 280 |
"database_status": "ready" if db_ready else "error",
|
| 281 |
+
"model_status": "ready" if model_ready else "error",
|
| 282 |
+
"cpu_optimization": {
|
| 283 |
+
"cpu_cores": CPU_CORES,
|
| 284 |
+
"llm_threads": LLM_THREADS,
|
| 285 |
+
"executor_workers": EXECUTOR_WORKERS
|
| 286 |
+
}
|
| 287 |
}
|
| 288 |
if not db_ready or not model_ready:
|
| 289 |
raise HTTPException(status_code=503, detail=status)
|
| 290 |
return status
|
| 291 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 292 |
@app.post("/chat")
|
| 293 |
async def chat(query: Query, request: Request):
|
| 294 |
adapter = get_logger_adapter(request)
|
| 295 |
question_lower = query.question.strip().lower()
|
| 296 |
|
| 297 |
+
# Greeting handling
|
| 298 |
greeting_keywords = ["hello", "hi", "hey", "what can you do", "who are you"]
|
| 299 |
if question_lower in greeting_keywords:
|
| 300 |
adapter.info(f"Handling a greeting or introductory query: '{query.question}'")
|
|
|
|
| 316 |
|
| 317 |
adapter.info(f"Received query: '{query.question}'")
|
| 318 |
|
| 319 |
+
# Query classification and search
|
| 320 |
query_type = classify_query_type(query.question)
|
| 321 |
adapter.info(f"Query classified as: {query_type}")
|
| 322 |
|
|
|
|
| 323 |
search_results = []
|
| 324 |
|
| 325 |
if query_type == "monetary":
|
|
|
|
| 326 |
amount = extract_monetary_amount(query.question)
|
| 327 |
if amount:
|
| 328 |
adapter.info(f"Extracted monetary amount: ₹{amount}")
|
|
|
|
| 331 |
search_results = monetary_results
|
| 332 |
adapter.info(f"Found {len(search_results)} results using monetary search")
|
| 333 |
|
|
|
|
| 334 |
if not search_results:
|
| 335 |
search_results = db.search_with_context(
|
| 336 |
query.question,
|
|
|
|
| 348 |
"answer": "Sorry, I could not find a relevant policy to answer that question. Please try rephrasing or ask about specific delegation limits, approval authorities, or procedures."
|
| 349 |
}
|
| 350 |
|
| 351 |
+
# Log search results with metadata
|
| 352 |
result_info = []
|
| 353 |
for i, result in enumerate(search_results):
|
| 354 |
metadata = result.get('metadata', {})
|
|
|
|
| 359 |
|
| 360 |
adapter.info(f"Search results: {' | '.join(result_info)}")
|
| 361 |
|
| 362 |
+
# Prepare context with metadata
|
| 363 |
context_chunks = []
|
| 364 |
for result in search_results[:TOP_K_CONTEXT]:
|
| 365 |
chunk_text = result['text']
|
| 366 |
metadata = result.get('metadata', {})
|
| 367 |
|
|
|
|
| 368 |
if metadata.get('section') or metadata.get('role'):
|
| 369 |
metadata_prefix = f"[Section: {metadata.get('section', 'N/A')}, Role: {metadata.get('role', 'N/A')}] "
|
| 370 |
chunk_text = metadata_prefix + chunk_text
|
|
|
|
| 372 |
context_chunks.append(chunk_text)
|
| 373 |
|
| 374 |
context = "\n---\n".join(context_chunks)
|
|
|
|
|
|
|
| 375 |
prompt = build_enhanced_prompt(query.question, context, query_type, search_results)
|
| 376 |
|
| 377 |
+
# Generate response
|
| 378 |
answer = "An error occurred while processing your request."
|
| 379 |
try:
|
| 380 |
adapter.info(f"Sending enhanced prompt to LLM for {query_type} query...")
|
|
|
|
| 384 |
)
|
| 385 |
adapter.info(f"LLM generation successful. Raw response: {raw_answer[:250]}...")
|
| 386 |
|
| 387 |
+
# Post-processing
|
| 388 |
if '|' in raw_answer:
|
| 389 |
adapter.info("Pipe separator found. Formatting response as a bulleted list.")
|
| 390 |
items = raw_answer.split('|')
|
| 391 |
cleaned_items = [f"• {item.strip()}" for item in items if item.strip()]
|
| 392 |
answer = "\n".join(cleaned_items)
|
| 393 |
else:
|
|
|
|
| 394 |
answer = raw_answer.strip()
|
| 395 |
|
|
|
|
| 396 |
if query_type == "monetary" and "₹" not in answer and extract_monetary_amount(query.question):
|
| 397 |
amount = extract_monetary_amount(query.question)
|
| 398 |
answer = f"For amounts of ₹{amount:,.0f}:\n\n{answer}"
|
|
|
|
| 410 |
"question": query.question,
|
| 411 |
"context_used": context,
|
| 412 |
"answer": answer,
|
| 413 |
+
"query_type": query_type,
|
| 414 |
"search_strategy": "monetary" if query_type == "monetary" and extract_monetary_amount(query.question) else "semantic_with_context"
|
| 415 |
}
|
| 416 |
|
|
|
|
| 429 |
adapter.info(json.dumps(feedback_log))
|
| 430 |
return {"status": "✅ Feedback recorded. Thank you!"}
|
| 431 |
|
| 432 |
+
# ✅ Cleanup on shutdown
|
| 433 |
+
@app.on_event("shutdown")
|
| 434 |
+
async def shutdown_event():
|
| 435 |
+
llm_executor.shutdown(wait=True)
|
| 436 |
+
logger.info("Thread pool executor shut down successfully.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|