# Load model directly from fastapi import FastAPI, Request import requests from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM ## create a new FASTAPI app instance app=FastAPI() # Initialize the text generation pipeline tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-small") model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-small") pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer) @app.get("/") def home(): return {"message":"Welcome to My LLM Custom Model"} def run_generation(text: str) -> str: output = pipe(text) return output[0]["generated_text"] # Define a function to handle the GET request at `/generate` @app.get("/generate") def generate(text:str): ## return the generate text in Json reposne return {"output": run_generation(text)} @app.post("/v1/chat/completions") async def chat_completions(request: Request): body = await request.json() messages = body.get("messages", []) if not messages: return {"error": "No messages provided"} prompt = messages[-1]["content"] generated_text = run_generation(prompt) return { "id": "hf-space", "object": "chat.completion", "choices": [ { "index": 0, "message": { "role": "assistant", "content": generated_text }, "finish_reason": "stop" } ] }