from fastapi import FastAPI, HTTPException from pydantic import BaseModel import transformers import torch import os from dotenv import load_dotenv # Load environment variables from .env file load_dotenv() API_TOKEN = os.getenv("HF_API_TOKEN") # Initialize the text-generation pipeline with authentication model_id = "mistralai/Mistral-7B-Instruct-v0.3" pipeline = transformers.pipeline( "text-generation", model=model_id, model_kwargs={"torch_dtype": torch.bfloat16}, device_map="auto", use_auth_token=API_TOKEN ) # Define the FastAPI app app = FastAPI() # Define the request model class Message(BaseModel): role: str content: str class ChatMessages(BaseModel): messages: list[Message] @app.get("/hello") def greet_json(): return {"Hello": "World!"} @app.post("/chat") async def chat(request: ChatMessages): # Prepare the input messages for the pipeline messages = [{"role": msg.role, "content": msg.content} for msg in request.messages] # Set the terminators terminators = [ pipeline.tokenizer.eos_token_id, pipeline.tokenizer.convert_tokens_to_ids("") ] # Generate response using the model try: # Mistral-7B-Instruct-v0.3 might require a slightly different format # Adjust this if needed according to the model's documentation outputs = pipeline( messages, max_new_tokens=256, eos_token_id=terminators, do_sample=True, temperature=0.6, top_p=0.9, ) response_text = outputs[0]["generated_text"] return {"response": response_text} except Exception as e: raise HTTPException(status_code=500, detail=str(e)) # Start the FastAPI app if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000)