| import os |
| from llama_cpp import Llama |
| from fastapi import FastAPI |
| from fastapi.middleware.cors import CORSMiddleware |
| from pydantic import BaseModel |
| import uvicorn |
|
|
| app = FastAPI() |
| app.add_middleware(CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]) |
|
|
| print("Loading Qwen 2.5 Coder 7B (4 GB)...", flush=True) |
| llm = Llama( |
| model_path="/app/models/qwen2.5-coder-7b-instruct-q4_k_m.gguf", |
| n_ctx=2048, |
| n_threads=4, |
| n_gpu_layers=0, |
| verbose=False, |
| ) |
| print("Qwen 7B loaded! Ready for inference.", flush=True) |
|
|
| class ChatRequest(BaseModel): |
| messages: list |
| max_tokens: int = 200 |
| temperature: float = 0.7 |
|
|
| @app.post("/v1/chat/completions") |
| async def chat(req: ChatRequest): |
| prompt = "" |
| for msg in req.messages: |
| role = msg.get("role", "user") |
| content = msg.get("content", "") |
| if role == "user": |
| prompt += f"<|im_start|>user\n{content}<|im_end|>\n" |
| elif role == "system": |
| prompt += f"<|im_start|>system\n{content}<|im_end|>\n" |
| elif role == "assistant": |
| prompt += f"<|im_start|>assistant\n{content}<|im_end|>\n" |
| prompt += "<|im_start|>assistant\n" |
| |
| output = llm(prompt, max_tokens=req.max_tokens, temperature=req.temperature, stop=["<|im_end|>"]) |
| text = output["choices"][0]["text"] |
| |
| return { |
| "choices": [{"message": {"role": "assistant", "content": text}}], |
| "model": "qwen2.5-coder-7b", |
| } |
|
|
| @app.get("/") |
| async def root(): |
| return {"status": "ready", "model": "Qwen 2.5 Coder 7B", "humaneval": "65%"} |
|
|
| @app.get("/health") |
| async def health(): |
| return {"status": "healthy"} |
|
|
| if __name__ == "__main__": |
| uvicorn.run(app, host="0.0.0.0", port=7860) |
|
|
