Fix: use +cpu tagged manylinux wheel from AIencoder (no CUDA deps)

app.py

@@ -2,49 +2,73 @@ import os
 import sys
 import subprocess
 
-# -------------------------------------------------------------
-# 1. Auto-bootstrap prebuilt manylinux llama-cpp-python
-#    (Installs in 3 seconds, zero build timeout, zero musl error)
-# -------------------------------------------------------------
-try:
-    from llama_cpp import Llama
-except ImportError:
-    print("Bootstrapping pre-built manylinux llama-cpp-python from Hugging Face...")
+# =============================================================
+# 1. Bootstrap: Install a KNOWN-GOOD prebuilt CPU-only wheel
+#    Versions >= 0.3.20 use "+cpu" tag = zero CUDA dependency
+# =============================================================
+def bootstrap_llama_cpp():
+    print("Bootstrapping llama-cpp-python (CPU-only, manylinux)...")
     from huggingface_hub import HfApi, hf_hub_download
 
     api = HfApi()
     files = api.list_repo_files("AIencoder/llama-cpp-wheels", repo_type="dataset")
 
     py_tag = f"cp{sys.version_info.major}{sys.version_info.minor}"
-    candidate = None
-    for p in files:
-        if p.endswith(".whl") and "linux/x86_64/cpu/" in p and (py_tag in p or "py3-none" in p):
-            candidate = p
-            break
-    if not candidate:
-        for p in files:
-            if p.endswith(".whl") and "linux/x86_64" in p and "cpu" in p and (py_tag in p or "py3-none" in p):
-                candidate = p
-                break
-
-    print(f"Installing precompiled wheel: {candidate}")
-    whl_path = hf_hub_download("AIencoder/llama-cpp-wheels", candidate, repo_type="dataset")
-    subprocess.check_call([sys.executable, "-m", "pip", "install", whl_path])
+
+    # Filter: linux/x86_64/cpu, version 0.3.x, "+cpu" in name, py3-none or matching cpXXX
+    candidates = [
+        p for p in files
+        if p.endswith(".whl")
+        and "linux/x86_64/cpu/" in p
+        and "/0.3." in p
+        and "+cpu" in p
+        and ("py3-none" in p or py_tag in p)
+    ]
+
+    if not candidates:
+        # Fallback: any 0.3.x cpu wheel matching our python
+        candidates = [
+            p for p in files
+            if p.endswith(".whl")
+            and "linux/x86_64/cpu/" in p
+            and "/0.3." in p
+            and ("py3-none" in p or py_tag in p)
+            and "cuda" not in p.lower()
+        ]
+
+    # Sort descending to pick latest version
+    candidates.sort(reverse=True)
+    if not candidates:
+        raise RuntimeError("No compatible CPU wheel found!")
+
+    chosen = candidates[0]
+    print(f"Selected wheel: {chosen}")
+    whl_path = hf_hub_download("AIencoder/llama-cpp-wheels", chosen, repo_type="dataset")
+    subprocess.check_call([
+        sys.executable, "-m", "pip", "install", "--force-reinstall", whl_path
+    ])
+    print("llama-cpp-python installed successfully!")
+
+try:
+    from llama_cpp import Llama
+    # Verify it actually loads (catches CUDA linkage errors)
+    print("llama-cpp-python already available.")
+except (ImportError, RuntimeError, OSError):
+    bootstrap_llama_cpp()
     from llama_cpp import Llama
-    print("llama-cpp-python ready!")
 
 from typing import List, Literal, Optional
 from pydantic import BaseModel, Field
 from fastapi import FastAPI, HTTPException, Header
 import gradio as gr
 
-# -------------------------------------------------------------
-# 2. Configuration & Load Qwen GGUF Model
-# -------------------------------------------------------------
+# =============================================================
+# 2. Load GGUF Model
+# =============================================================
 MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf"
 API_KEY = os.getenv("API_KEY", "commitcon-secret-2026")
 
-print(f"Loading {MODEL_PATH} via llama-cpp ...")
+print(f"Loading {MODEL_PATH} via llama-cpp-python ...")
 llm = Llama(
     model_path=MODEL_PATH,
     n_ctx=4096,
@@ -53,9 +77,9 @@ llm = Llama(
 )
 print("Model loaded successfully!")
 
-# -------------------------------------------------------------
-# 3. FastAPI Service Endpoint
-# -------------------------------------------------------------
+# =============================================================
+# 3. FastAPI Endpoints
+# =============================================================
 app = FastAPI(title="CommitCon Sherpa AI Copilot")
 
 class HistoryMessage(BaseModel):
@@ -69,9 +93,10 @@ class ChatRequest(BaseModel):
     history: List[HistoryMessage] = Field(default_factory=list)
 
 SYSTEM_PROMPT = (
-    "You are Sherpa, an AI driving assistant for the CommitCon ride-hailing repositioning platform in Bengaluru. "
-    "Your job is to provide concise, direct, helpful recommendations to drivers about where to reposition for high demand, "
-    "earnings, and low traffic delays. Keep your advice practical, realistic, and focused on maximizing their earnings."
+    "You are Sherpa, an AI driving assistant for the CommitCon ride-hailing "
+    "repositioning platform in Bengaluru. Provide concise, direct, helpful "
+    "recommendations to drivers about where to reposition for high demand, "
+    "earnings, and low traffic delays."
 )
 
 def run_inference(prompt: str, history: List[dict] = None) -> str:
@@ -80,11 +105,8 @@ def run_inference(prompt: str, history: List[dict] = None) -> str:
         for m in history:
             messages.append({"role": m["role"], "content": m["content"]})
     messages.append({"role": "user", "content": prompt})
-
     response = llm.create_chat_completion(
-        messages=messages,
-        temperature=0.7,
-        max_tokens=512,
+        messages=messages, temperature=0.7, max_tokens=512
     )
     return response["choices"][0]["message"]["content"]
 
@@ -96,20 +118,16 @@ def root():
 def chat_endpoint(payload: ChatRequest, x_api_key: Optional[str] = Header(None)):
     if API_KEY and x_api_key != API_KEY:
         raise HTTPException(status_code=401, detail="Unauthorized: Invalid API Key")
-
     formatted_history = [{"role": m.role, "content": m.content} for m in payload.history]
     answer = run_inference(payload.driver_prompt, formatted_history)
-
     return {
-        "status": "success",
-        "answer": answer,
-        "model": "project-driver-qwen-gguf",
-        "zone_context": payload.current_zone_id
+        "status": "success", "answer": answer,
+        "model": "project-driver-qwen-gguf", "zone_context": payload.current_zone_id
     }
 
-# -------------------------------------------------------------
-# 4. Mount Gradio Playground for testing
-# -------------------------------------------------------------
+# =============================================================
+# 4. Gradio Chat UI
+# =============================================================
 def gradio_chat(user_message, history):
     hist_formatted = []
     if history:
@@ -122,7 +140,6 @@ def gradio_chat(user_message, history):
 demo = gr.ChatInterface(
     fn=gradio_chat,
     title="🧭 CommitCon Sherpa Driver Copilot",
-    description="Fine-tuned Qwen 0.8B Q4_K_M AI Driver Copilot for ride repositioning."
+    description="Fine-tuned Qwen 0.8B Q4_K_M — llama-cpp-python CPU inference"
 )
-
 app = gr.mount_gradio_app(app, demo, path="/")