File size: 452 Bytes
38352c3 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 | # app.py
from fastapi import FastAPI
from llama_cpp import Llama
from huggingface_hub import hf_hub_download
app = FastAPI()
model_path = hf_hub_download(
repo_id="bartowski/Phi-3.5-mini-instruct-GGUF",
filename="Phi-3.5-mini-instruct-Q4_K_M.gguf"
)
llm = Llama(model_path=model_path, n_ctx=4096)
@app.post("/generate")
def generate(prompt: str):
output = llm(prompt, max_tokens=512)
return {"response": output["choices"][0]["text"]} |