fix(app): use Llama.from_pretrained to auto-download GGUF from hub

app.py

@@ -38,15 +38,15 @@ try:
     LOCAL_7B_P95 = 37.47 
     
     if LOCAL_7B_P95 <= 14.0:
-        local_model_path = "qwen2.5-7b-instruct-q4_k_m.gguf"
+        local_repo = "bartowski/Qwen2.5-7B-Instruct-GGUF"`r`n        local_file = "Qwen2.5-7B-Instruct-Q4_K_M.gguf"
         LOCAL_MAX_TOKENS = 150
         logger.info(f"Using 7b for local tier since p95 latency ({LOCAL_7B_P95}s) is within 14s budget.")
     else:
-        local_model_path = "qwen2.5-3b-instruct-q4_k_m.gguf"
+        local_repo = "Qwen/Qwen2.5-3B-Instruct-GGUF"`r`n        local_file = "qwen2.5-3b-instruct-q4_k_m.gguf"
         LOCAL_MAX_TOKENS = 250
         logger.info(f"7b p95 {LOCAL_7B_P95}s exceeds 14s budget, using 3b instead.")
         
-    local_model = Llama(model_path=local_model_path, n_ctx=2048, n_threads=os.cpu_count(), verbose=False)
+    local_model = Llama.from_pretrained(repo_id=local_repo, filename=local_file, n_ctx=2048, n_threads=os.cpu_count(), verbose=False)
 except Exception as e:
     logger.warning(f"Failed to load local model: {e}")
     local_model = None
@@ -330,3 +330,4 @@ async def reply(req: Dict[str, Any]):
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8080)
+