File size: 3,483 Bytes
2a89b99 d278d78 c2da8e7 88fd288 e0f9b1a d278d78 c2da8e7 6def678 c2da8e7 f1fe51a e0f9b1a c2da8e7 6def678 c2da8e7 88fd288 73dad05 88fd288 e0f9b1a 88fd288 e0f9b1a 88fd288 8b4ea9c 88fd288 19bcadf 88fd288 8b4ea9c 73dad05 88fd288 e0f9b1a 8b4ea9c e0f9b1a 2a89b99 e0f9b1a 2a89b99 e0f9b1a 2a89b99 e0f9b1a 2a89b99 e0f9b1a 2a89b99 e0f9b1a 2a89b99 e0f9b1a 88fd288 d278d78 88fd288 c2da8e7 e0f9b1a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 | import json
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from functools import lru_cache
from clients.groq_client import GroqClient
from fastapi.responses import StreamingResponse
app = FastAPI()
# Allowed origins
origins = [
"http://localhost:4200",
"https://mosesmangabo.vercel.app",
"https://chatbot-bridge-service.onrender.com",
"https://eng-musa-chatbotbridge.hf.space",
]
app.add_middleware(
CORSMiddleware,
allow_origins=origins,
allow_credentials=True,
allow_methods=["*"], # allow all HTTP methods (GET, POST, ...)
allow_headers=["*"], # allow all headers
)
class QueryRequest(BaseModel):
query: str
ip: str
# Lazy singleton for GroqClient
@lru_cache(maxsize=1)
def get_groq_client():
return GroqClient()
# Non - Streaming
# ============================================
@app.post("/chat")
async def chat_endpoint(request: QueryRequest):
try:
groq_client = get_groq_client()
response, model = groq_client.ask(request.query)
if response.startswith("Error while"):
return {
"status_code": 500,
"response": response,
"model": model,
"next_questions": [],
}
return {
"status_code": 200,
"response": response,
"model": model,
"next_questions": groq_client.get_next_questions(request.ip),
}
except Exception as e:
return {
"status_code": 500,
"response": f"Internal error: {e}",
"model": None,
"next_questions": [],
}
# Streaming
# ============================================
@app.post("/chat-stream")
async def chat_stream_endpoint(request: QueryRequest):
"""Streaming endpoint that returns chunks in real-time with a consistent structure"""
try:
groq_client = get_groq_client()
def generate():
try:
response = ""
for chunk in groq_client.ask_stream(request.query):
response += chunk
yield f"data: {json.dumps({'status_code': 200, 'response': chunk, 'next_questions': []})}\n\n"
if response.startswith("Error while"):
yield f"data: {json.dumps({'status_code': 500, 'response': response, 'next_questions': []})}\n\n"
else:
yield f"data: {json.dumps({'status_code': 200, 'response': response, 'next_questions': []})}\n\n"
yield f"data: {json.dumps({'status_code': 200, 'response': '[DONE]', 'next_questions': groq_client.get_next_questions(request.ip)})}\n\n"
except Exception as e:
yield f"data: {json.dumps({'status_code': 500, 'response': f'Error: {str(e)}', 'next_questions': []})}\n\n"
return StreamingResponse(
generate(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Connection": "keep-alive",
"X-Accel-Buffering": "no", # Disable nginx buffering
},
)
except Exception as e:
return {
"status_code": 500,
"response": f"Internal error: {e}",
"next_questions": [],
}
@app.get("/")
def healthcheck():
return {"status": "ok"}
# python -m uvicorn main:app --reload
|