DockerTest1 / app.py
NubeEra's picture
V3
fc85d13 verified
Raw History Blame Contribute Delete
1.54 kB
# Load model directly
from fastapi import FastAPI, Request
import requests
from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
## create a new FASTAPI app instance
app=FastAPI()
# Initialize the text generation pipeline
tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-small")
model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-small")
pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer)
@app.get("/")
def home():
return {"message":"Welcome to My LLM Custom Model"}
def run_generation(text: str) -> str:
output = pipe(text)
return output[0]["generated_text"]
# Define a function to handle the GET request at `/generate`
@app.get("/generate")
def generate(text:str):
## return the generate text in Json reposne
return {"output": run_generation(text)}
@app.post("/v1/chat/completions")
async def chat_completions(request: Request):
body = await request.json()
messages = body.get("messages", [])
if not messages:
return {"error": "No messages provided"}
prompt = messages[-1]["content"]
generated_text = run_generation(prompt)
return {
"id": "hf-space",
"object": "chat.completion",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": generated_text
},
"finish_reason": "stop"
}
]
}