Spaces / app.py
Idhammz's picture
Update app.py
e001160 verified
Raw History Blame Contribute Delete
1.85 kB
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
import transformers
import torch
import os
from dotenv import load_dotenv
# Load environment variables from .env file
load_dotenv()
API_TOKEN = os.getenv("HF_API_TOKEN")
# Initialize the text-generation pipeline with authentication
model_id = "mistralai/Mistral-7B-Instruct-v0.3"
pipeline = transformers.pipeline(
"text-generation",
model=model_id,
model_kwargs={"torch_dtype": torch.bfloat16},
device_map="auto",
use_auth_token=API_TOKEN
)
# Define the FastAPI app
app = FastAPI()
# Define the request model
class Message(BaseModel):
role: str
content: str
class ChatMessages(BaseModel):
messages: list[Message]
@app.get("/hello")
def greet_json():
return {"Hello": "World!"}
@app.post("/chat")
async def chat(request: ChatMessages):
# Prepare the input messages for the pipeline
messages = [{"role": msg.role, "content": msg.content} for msg in request.messages]
# Set the terminators
terminators = [
pipeline.tokenizer.eos_token_id,
pipeline.tokenizer.convert_tokens_to_ids("")
]
# Generate response using the model
try:
# Mistral-7B-Instruct-v0.3 might require a slightly different format
# Adjust this if needed according to the model's documentation
outputs = pipeline(
messages,
max_new_tokens=256,
eos_token_id=terminators,
do_sample=True,
temperature=0.6,
top_p=0.9,
)
response_text = outputs[0]["generated_text"]
return {"response": response_text}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
# Start the FastAPI app
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)