| from huggingface_hub import hf_hub_download |
| from llama_cpp import Llama |
| from fastapi import FastAPI |
| from pydantic import BaseModel |
| import uvicorn |
|
|
| MODEL_PATH = hf_hub_download( |
| repo_id="bartowski/Qwen2.5-Coder-3B-Instruct-abliterated-GGUF", |
| filename="Qwen2.5-Coder-3B-Instruct-abliterated-Q4_K_M.gguf" |
| ) |
|
|
| llm = Llama(model_path=MODEL_PATH, n_ctx=2048, n_threads=2, chat_format="chatml") |
|
|
| app = FastAPI() |
|
|
| class ChatRequest(BaseModel): |
| model: str = "local" |
| messages: list |
| temperature: float = 0.7 |
| max_tokens: int = 1024 |
|
|
| @app.get("/health") |
| def health(): |
| return {"status": "ok"} |
|
|
| @app.post("/v1/chat/completions") |
| def chat(req: ChatRequest): |
| out = llm.create_chat_completion( |
| messages=req.messages, |
| temperature=req.temperature, |
| max_tokens=req.max_tokens |
| ) |
| return out |
|
|
| if __name__ == "__main__": |
| uvicorn.run(app, host="0.0.0.0", port=7860) |
