| # app.py | |
| from fastapi import FastAPI | |
| from llama_cpp import Llama | |
| from huggingface_hub import hf_hub_download | |
| app = FastAPI() | |
| model_path = hf_hub_download( | |
| repo_id="bartowski/Phi-3.5-mini-instruct-GGUF", | |
| filename="Phi-3.5-mini-instruct-Q4_K_M.gguf" | |
| ) | |
| llm = Llama(model_path=model_path, n_ctx=4096) | |
| def generate(prompt: str): | |
| output = llm(prompt, max_tokens=512) | |
| return {"response": output["choices"][0]["text"]} |