Update app.py

app.py

@@ -8,6 +8,7 @@ from llama_cpp import Llama
 
 app = FastAPI(title="Qwen 2.5 Coder OpenAI Server")
 
+# Enable CORS for local VS Code extension connections (Cline, Continue, Cursor)
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -20,13 +21,14 @@ MODEL_REPO = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF"
 MODEL_FILE = "qwen2.5-coder-7b-instruct-q4_k_m.gguf"
 
 print("Checking and downloading Qwen 2.5 Coder 7B GGUF model...")
+# Downloads dynamically on container boot time via the HF library (cached across container restarts)
 model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
 
 print(f"Loading model into memory from: {model_path}")
 llm = Llama(
     model_path=model_path,
-    n_ctx=4096,      
-    n_threads=2,     
+    n_ctx=4096,      # Safely fits within 16GB memory bounds
+    n_threads=2,     # Perfectly matches your 2 vCPU resource cap
     n_batch=512,
     verbose=False
 )
@@ -39,6 +41,7 @@ async def chat_completions(request: Request):
     temperature = body.get("temperature", 0.2)
     max_tokens = body.get("max_tokens", 1024)
 
+    # Convert standard OpenAI messages array into Qwen ChatML formatting
     formatted_prompt = ""
     for msg in messages:
         role = msg.get("role", "user")
@@ -46,6 +49,7 @@ async def chat_completions(request: Request):
         formatted_prompt += f"<|im_start|>{role}\n{content}<|im_end|>\n"
     formatted_prompt += "<|im_start|>assistant\n"
 
+    # Synchronous processing avoids thread-locking/stalling on weak CPUs
     response = llm(
         formatted_prompt,
         max_tokens=max_tokens,
@@ -55,6 +59,7 @@ async def chat_completions(request: Request):
 
     text_output = response["choices"]["text"]
     
+    # Construct structured JSON compliant with OpenAI specs
     return JSONResponse(content={
         "id": f"chatcmpl-{int(time.time())}",
         "object": "chat.completion",
@@ -83,3 +88,4 @@ def index():
         "endpoint": "/v1/chat/completions"
     }
 
+