File size: 452 Bytes
38352c3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# app.py
from fastapi import FastAPI
from llama_cpp import Llama
from huggingface_hub import hf_hub_download

app = FastAPI()

model_path = hf_hub_download(
    repo_id="bartowski/Phi-3.5-mini-instruct-GGUF",
    filename="Phi-3.5-mini-instruct-Q4_K_M.gguf"
)
llm = Llama(model_path=model_path, n_ctx=4096)

@app.post("/generate")
def generate(prompt: str):
    output = llm(prompt, max_tokens=512)
    return {"response": output["choices"][0]["text"]}