Switch to ctransformers for pure glibc compatibility

app.py

@@ -3,20 +3,21 @@ from typing import List, Literal, Optional
 from pydantic import BaseModel, Field
 from fastapi import FastAPI, HTTPException, Header
 import gradio as gr
-from llama_cpp import Llama
+from ctransformers import AutoModelForCausalLM
 
 # -------------------------------------------------------------
-# Configuration & Secret
+# Configuration & Load GGUF Model
 # -------------------------------------------------------------
 MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf"
 API_KEY = os.getenv("API_KEY", "commitcon-secret-2026")
 
-print("Loading GGUF model via llama-cpp ...")
-llm = Llama(
-    model_path=MODEL_PATH,
-    n_ctx=4096,
-    n_threads=2,
-    verbose=False
+print("Loading GGUF model via ctransformers ...")
+llm = AutoModelForCausalLM.from_pretrained(
+    ".",
+    model_file=MODEL_PATH,
+    model_type="qwen2",
+    context_length=4096,
+    threads=2
 )
 print("Model loaded successfully!")
 
@@ -42,18 +43,20 @@ SYSTEM_PROMPT = (
 )
 
 def run_inference(prompt: str, history: List[dict] = None) -> str:
-    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+    # Build prompt formatted for chat
+    formatted_prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
     if history:
         for m in history:
-            messages.append({"role": m["role"], "content": m["content"]})
-    messages.append({"role": "user", "content": prompt})
+            formatted_prompt += f"<|im_start|>{m['role']}\n{m['content']}<|im_end|>\n"
+    formatted_prompt += f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
 
-    response = llm.create_chat_completion(
-        messages=messages,
+    response = llm(
+        formatted_prompt,
+        max_new_tokens=512,
         temperature=0.7,
-        max_tokens=512,
+        stop=["<|im_end|>", "<|endoftext|>"]
     )
-    return response["choices"][0]["message"]["content"]
+    return response.strip()
 
 @app.get("/")
 def root():
@@ -75,7 +78,7 @@ def chat_endpoint(payload: ChatRequest, x_api_key: Optional[str] = Header(None))
     }
 
 # -------------------------------------------------------------
-# Gradio UI Mount (allows testing directly on Hugging Face)
+# Gradio UI Mount (playground on Hugging Face)
 # -------------------------------------------------------------
 def gradio_chat(user_message, history):
     hist_formatted = []
@@ -92,5 +95,4 @@ demo = gr.ChatInterface(
     description="Fine-tuned Qwen 0.8B Q4_K_M AI Driver Copilot for ride repositioning."
 )
 
-# Mount Gradio onto the FastAPI app
 app = gr.mount_gradio_app(app, demo, path="/")

requirements.txt

@@ -1,4 +1,3 @@
---extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
-llama-cpp-python==0.2.90
+ctransformers>=0.2.27
 fastapi>=0.115.0
 pydantic>=2.0.0