Update app.py

app.py

@@ -1,444 +1,106 @@
-"""
-OpenAI-compatible FastAPI + Gradio front-end for Qwen3.6-35B-A3B-MTP,
-running on the real llama.cpp `llama-server` binary so that native
-Multi-Token Prediction (MTP) speculative decoding is actually used.
-
-Designed for Hugging Face Spaces.
-
-IMPORTANT:
-- llama-server is started inside FastAPI lifespan.
-- No top-level llama-server startup.
-- No uvicorn.run() is required on HF Spaces.
-- No atexit handler is required.
-"""
-
-import os
+Import os
+import sys
 import time
-import json
-import shutil
-import socket
-import stat
-import tarfile
-import tempfile
-import subprocess
 import re
+import json
+import multiprocessing
+from typing import List, Dict, Any, Optional, Union, AsyncGenerator
 
-from contextlib import asynccontextmanager
-from typing import List, Dict, Any, Optional, AsyncGenerator
-
-import httpx
-import gradio as gr
-
-from fastapi import FastAPI, HTTPException
+# FastAPI & Pydantic imports
+from fastapi import FastAPI, Request, HTTPException
 from fastapi.responses import StreamingResponse, JSONResponse
 from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel, Field
+
+# ---------------------------------------------------------------------------
+# ⚡ 1. CPU & THREADING OPTIMIZATIONS FOR MoE
+# ---------------------------------------------------------------------------
+# MoE models are heavily bottlenecked by CPU RAM bandwidth during expert routing.
+# Locking threads to physical cores avoids hyperthreading thrashing.
+try:
+    import psutil
+    PHYSICAL_CORES = psutil.cpu_count(logical=False) or max(1, multiprocessing.cpu_count() // 2)
+except Exception:
+    PHYSICAL_CORES = max(1, multiprocessing.cpu_count() // 2)
+
+os.environ["OMP_NUM_THREADS"] = str(PHYSICAL_CORES)
+os.environ["MKL_NUM_THREADS"] = str(PHYSICAL_CORES)
+os.environ["OPENBLAS_NUM_THREADS"] = str(PHYSICAL_CORES)
+os.environ["OMP_PROC_BIND"] = "TRUE"
+
+try:
+    p = psutil.Process()
+    if sys.platform == "win32":
+        p.nice(psutil.HIGH_PRIORITY_CLASS)
+    else:
+        p.nice(-15)
+except Exception:
+    pass
 
-from pydantic import BaseModel
+import gradio as gr
+import llama_cpp
+from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
+# ---------------------------------------------------------------------------
+# ⚡ 2. MODEL DOWNLOAD & LLAMA-CPP MoE INITIALIZATION
+# ---------------------------------------------------------------------------
+NUMA_STRATEGY = getattr(llama_cpp, "GGML_NUMA_STRATEGY_DISTRIBUTE", False)
 
-# ===========================================================================
-# 0. CONFIG
-# ===========================================================================
-
-REPO_ID = "unsloth/Qwen3.6-35B-A3B-MTP-GGUF"
-MODEL_FILENAME = "Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf"
-
-MODEL_NAME = "arush/jarush"
-
-LLAMA_SERVER_HOST = "127.0.0.1"
-LLAMA_SERVER_PORT = 8910
-PUBLIC_PORT = 7860
-
-CTX_SIZE = 4096
-N_BATCH = 512
-N_UBATCH = 512
-SPEC_DRAFT_N_MAX = 3
-
-LLAMA_INSTALL_DIR = os.path.expanduser("~/llama.cpp-prebuilt")
-LLAMA_SERVER_BIN = os.path.join(
-    LLAMA_INSTALL_DIR,
-    "build",
-    "bin",
-    "llama-server",
+print("Downloading MoE model GGUF...")
+model_path = hf_hub_download(
+    repo_id='unsloth/Qwen3.6-35B-A3B-MTP-GGUF',
+    filename="Qwen3.6-35B-A3B-UD-IQ3_XXS.gguf",
 )
-
-GITHUB_API_LATEST_RELEASE = (
-    "https://api.github.com/repos/ggml-org/llama.cpp/releases/latest"
+print("Model downloaded successfully!")
+
+# Load GGUF model optimized for CPU MoE execution
+llm = Llama(
+    model_path=model_path,
+    n_gpu_layers=0,             # 0 for CPU execution
+    n_ctx=4096,
+    
+    # Threading optimizations
+    n_threads=4,       
+    n_threads_batch=PHYSICAL_CORES, 
+    numa=NUMA_STRATEGY,
+    
+    # ⚡ MTP Configuration (Multi-Token Prediction)
+                    # Instructs llama.cpp to predict 3 extra tokens using native MTP modules
+    
+    # MoE Cache & RAM Bandwidth Optimizations:
+    # 1. Quantized KV Cache (Q4_0) saves RAM and frees CPU cache space for routing
+    type_k=getattr(llama_cpp, "GGML_TYPE_Q4_0", 2),
+    type_v=getattr(llama_cpp, "GGML_TYPE_Q4_0", 2),
+    
+    # 2. Smaller ubatch prevents L2/L3 cache eviction during active expert evaluation
+    n_batch=512,    
+    n_ubatch=512,   
+    n_draft=2,
+    use_mlock=False,
+    use_mmap=True,
+    flash_attn=True,
 )
-PREBUILT_ASSET_SUFFIX = "-bin-ubuntu-x64.tar.gz"
-
-PHYSICAL_CORES = max(1, (os.cpu_count() or 4) // 2)
-
-
-# ===========================================================================
-# 1. FETCH PREBUILT llama-server
-# ===========================================================================
-
-def ensure_llama_server_downloaded() -> None:
-    if (
-        os.path.isfile(LLAMA_SERVER_BIN)
-        and os.access(LLAMA_SERVER_BIN, os.X_OK)
-    ):
-        print(
-            f"[fetch] Found existing prebuilt llama-server at "
-            f"{LLAMA_SERVER_BIN}"
-        )
-        return
-
-    print("[fetch] Looking up latest llama.cpp release ...")
-
-    with httpx.Client(timeout=30, follow_redirects=True) as client:
-        resp = client.get(GITHUB_API_LATEST_RELEASE)
-        resp.raise_for_status()
-        release = resp.json()
-
-        asset = next(
-            (
-                a
-                for a in release.get("assets", [])
-                if a["name"].endswith(PREBUILT_ASSET_SUFFIX)
-            ),
-            None,
-        )
-
-        if asset is None:
-            raise RuntimeError(
-                f"No asset ending in '{PREBUILT_ASSET_SUFFIX}' found."
-            )
-
-        print(
-            f"[fetch] Downloading prebuilt binary: "
-            f"{asset['name']} ({asset['size'] / 1e6:.1f} MB) ..."
-        )
-
-        os.makedirs(LLAMA_INSTALL_DIR, exist_ok=True)
-
-        tarball_path = os.path.join(
-            tempfile.gettempdir(),
-            asset["name"],
-        )
-
-        with client.stream(
-            "GET",
-            asset["browser_download_url"],
-        ) as r:
-            r.raise_for_status()
-
-            with open(tarball_path, "wb") as f:
-                for chunk in r.iter_bytes(chunk_size=1 << 20):
-                    f.write(chunk)
-
-    print("[fetch] Extracting ...")
-
-    build_bin_dir = os.path.join(
-        LLAMA_INSTALL_DIR,
-        "build",
-        "bin",
-    )
-    os.makedirs(build_bin_dir, exist_ok=True)
-
-    with tarfile.open(tarball_path, "r:gz") as tf:
-        tf.extractall(LLAMA_INSTALL_DIR)
-
-    if not os.path.isfile(LLAMA_SERVER_BIN):
-        found = None
-
-        for root, _, files in os.walk(LLAMA_INSTALL_DIR):
-            if "llama-server" in files:
-                found = os.path.join(root, "llama-server")
-                break
-
-        if found is None:
-            raise RuntimeError(
-                "Extracted llama.cpp archive but couldn't locate "
-                "a 'llama-server' binary."
-            )
-
-        os.makedirs(
-            os.path.dirname(LLAMA_SERVER_BIN),
-            exist_ok=True,
-        )
-
-        shutil.copy2(found, LLAMA_SERVER_BIN)
-
-        src_dir = os.path.dirname(found)
-        dst_dir = os.path.dirname(LLAMA_SERVER_BIN)
-
-        for fname in os.listdir(src_dir):
-            if fname.startswith("lib") and ".so" in fname:
-                src = os.path.join(src_dir, fname)
-                dst = os.path.join(dst_dir, fname)
-
-                try:
-                    shutil.copy2(src, dst)
-                except Exception as e:
-                    print(
-                        f"[fetch] Warning: could not copy {fname}: {e}"
-                    )
-
-    st = os.stat(LLAMA_SERVER_BIN)
-
-    os.chmod(
-        LLAMA_SERVER_BIN,
-        st.st_mode
-        | stat.S_IEXEC
-        | stat.S_IXGRP
-        | stat.S_IXOTH,
-    )
-
-    try:
-        os.remove(tarball_path)
-    except OSError:
-        pass
-
-    print(f"[fetch] Ready: {LLAMA_SERVER_BIN}")
-
-
-# ===========================================================================
-# 2. DOWNLOAD MTP GGUF
-# ===========================================================================
-
-def ensure_model_downloaded() -> str:
-    print(
-        f"[download] Fetching {MODEL_FILENAME} from {REPO_ID} ..."
-    )
-
-    path = hf_hub_download(
-        repo_id=REPO_ID,
-        filename=MODEL_FILENAME,
-    )
-
-    print(f"[download] Model ready at {path}")
-
-    return path
-
-
-# ===========================================================================
-# 3. LLAMA-SERVER PROCESS MANAGEMENT
-# ===========================================================================
-
-_llama_proc: Optional[subprocess.Popen] = None
-
-
-def _port_is_open(
-    host: str,
-    port: int,
-    timeout: float = 1.0,
-) -> bool:
-    try:
-        with socket.create_connection(
-            (host, port),
-            timeout=timeout,
-        ):
-            return True
-    except OSError:
-        return False
-
-
-def start_llama_server(
-    model_path: str,
-) -> subprocess.Popen:
-    global _llama_proc
-
-    cmd = [
-        LLAMA_SERVER_BIN,
-        "-m",
-        model_path,
-        "--host",
-        LLAMA_SERVER_HOST,
-        "--port",
-        str(LLAMA_SERVER_PORT),
-        "-c",
-        str(CTX_SIZE),
-        "-b",
-        str(N_BATCH),
-        "-ub",
-        str(N_UBATCH),
-        "-t",
-        str(PHYSICAL_CORES),
-        "-ngl",
-        "0",
-        "--flash-attn",
-        "on",
-        "--cache-type-k",
-        "q4_0",
-        "--cache-type-v",
-        "q4_0",
-        "--spec-type",
-        "draft-mtp",
-        "--spec-draft-n-max",
-        str(SPEC_DRAFT_N_MAX),
-        "--jinja",
-        "--alias",
-        MODEL_NAME,
-        "--metrics",
-    ]
-
-    print("[llama-server] launching:")
-    print(" ".join(cmd))
-
-    proc = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        text=True,
-        bufsize=1,
-    )
-
-    _llama_proc = proc
-
-    deadline = time.time() + 600
-
-    while time.time() < deadline:
-        if proc.poll() is not None:
-            remaining = proc.stdout.read() if proc.stdout else ""
-
-            raise RuntimeError(
-                "llama-server exited early "
-                f"with code {proc.returncode}.\n"
-                f"Output:\n{remaining}"
-            )
-
-        if _port_is_open(
-            LLAMA_SERVER_HOST,
-            LLAMA_SERVER_PORT,
-        ):
-            print("[llama-server] ready.")
-            return proc
-
-        time.sleep(2)
-
-    print(
-        "[llama-server] startup timeout; terminating process..."
-    )
-
-    proc.terminate()
-
-    try:
-        proc.wait(timeout=15)
-    except subprocess.TimeoutExpired:
-        proc.kill()
-
-    raise TimeoutError(
-        "llama-server did not become ready within 600 seconds."
-    )
-
-
-def stop_llama_server() -> None:
-    global _llama_proc
-
-    if (
-        _llama_proc
-        and _llama_proc.poll() is None
-    ):
-        print(
-            "[llama-server] shutting down subprocess..."
-        )
-
-        _llama_proc.terminate()
-
-        try:
-            _llama_proc.wait(timeout=15)
-
-        except subprocess.TimeoutExpired:
-            print(
-                "[llama-server] graceful shutdown timed out; killing..."
-            )
-
-            _llama_proc.kill()
-
-            try:
-                _llama_proc.wait(timeout=5)
-            except subprocess.TimeoutExpired:
-                pass
-
-    _llama_proc = None
-
-
-# ===========================================================================
-# 4. FASTAPI LIFESPAN
-# ===========================================================================
-
-LLAMA_BASE_URL = ""
 
-_http_client: Optional[httpx.AsyncClient] = None
-
-
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    global _llama_proc
-    global _http_client
-    global LLAMA_BASE_URL
-
-    print("[lifespan] Starting application...")
-
-    try:
-        ensure_llama_server_downloaded()
-
-        model_path = ensure_model_downloaded()
-
-        _llama_proc = start_llama_server(model_path)
-
-        LLAMA_BASE_URL = (
-            f"http://{LLAMA_SERVER_HOST}:{LLAMA_SERVER_PORT}"
-        )
-
-        _http_client = httpx.AsyncClient(
-            base_url=LLAMA_BASE_URL,
-            timeout=None,
-        )
-
-        print("[lifespan] llama-server is ready.")
-        print("[lifespan] Application startup complete.")
-
-        yield
-
-    finally:
-        print("[lifespan] Shutting down...")
-
-        if _http_client is not None:
-            try:
-                await _http_client.aclose()
-            except Exception as e:
-                print(
-                    f"[lifespan] HTTP client shutdown warning: {e}"
-                )
-
-            _http_client = None
-
-        stop_llama_server()
-
-        print("[lifespan] Shutdown complete.")
-
-
-# ===========================================================================
-# 5. FASTAPI APPLICATION
-# ===========================================================================
+MODEL_NAME = "arush/jarush"
 
-app = FastAPI(
-    title="OpenAI-Compatible MTP Proxy Server",
-    version="2.0.0",
-    lifespan=lifespan,
-)
+# ---------------------------------------------------------------------------
+# ⚡ 3. FASTAPI INITIALIZATION & OPENAI PYDANTIC SCHEMAS
+# ---------------------------------------------------------------------------
+app = FastAPI(title="OpenAI-Compatible CPU MoE Server", version="1.0.0")
 
+# Enable CORS for external web frontend requests
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
-    allow_credentials=False,
+    allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
 )
 
-
-# ===========================================================================
-# 6. OPENAI-COMPATIBLE API MODELS
-# ===========================================================================
-
 class ChatMessage(BaseModel):
     role: str
-    content: Any
-
+    content: Union[str, List[Dict[str, Any]]]
 
 class ChatCompletionRequest(BaseModel):
     model: Optional[str] = MODEL_NAME
@@ -448,13 +110,12 @@ class ChatCompletionRequest(BaseModel):
     max_tokens: Optional[int] = 512
     stream: Optional[bool] = False
 
-
-# ===========================================================================
-# 7. OPENAI-COMPATIBLE ENDPOINTS
-# ===========================================================================
-
+# ---------------------------------------------------------------------------
+# ⚡ 4. OPENAI-COMPATIBLE REST ENDPOINTS
+# ---------------------------------------------------------------------------
 @app.get("/v1/models")
 async def list_models():
+    """OpenAI standard /v1/models endpoint."""
     return {
         "object": "list",
         "data": [
@@ -467,332 +128,133 @@ async def list_models():
         ],
     }
 
-
-async def _proxy_stream(
-    payload: Dict[str, Any],
-) -> AsyncGenerator[bytes, None]:
-    if _http_client is None:
-        raise RuntimeError("llama-server is not ready.")
-
-    async with _http_client.stream(
-        "POST",
-        "/v1/chat/completions",
-        json=payload,
-    ) as resp:
-        resp.raise_for_status()
-
-        async for chunk in resp.aiter_bytes():
-            yield chunk
-
+async def openai_stream_generator(messages_dict: list, max_tokens: int, temperature: float, top_p: float):
+    """Generates standard SSE streaming tokens for OpenAI client libraries."""
+    for chunk in llm.create_chat_completion(
+        messages=messages_dict,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        stream=True,
+    ):
+        yield f"data: {json.dumps(chunk)}\n\n"
+    yield "data: [DONE]\n\n"
 
 @app.post("/v1/chat/completions")
-async def chat_completions(
-    req: ChatCompletionRequest,
-):
-    if _http_client is None:
-        raise HTTPException(
-            status_code=503,
-            detail="llama-server is still starting.",
-        )
-
-    payload = {
-        "model": req.model,
-        "messages": [
-            m.model_dump()
-            for m in req.messages
-        ],
-        "temperature": req.temperature,
-        "top_p": req.top_p,
-        "max_tokens": req.max_tokens,
-        "stream": req.stream,
-    }
-
+async def chat_completions(req: ChatCompletionRequest):
+    """OpenAI standard /v1/chat/completions endpoint."""
     try:
+        # Convert pydantic messages to standard list of dicts
+        messages_dict = [{"role": m.role, "content": m.content} for m in req.messages]
+        
         if req.stream:
             return StreamingResponse(
-                _proxy_stream(payload),
+                openai_stream_generator(
+                    messages_dict=messages_dict,
+                    max_tokens=req.max_tokens,
+                    temperature=req.temperature,
+                    top_p=req.top_p,
+                ),
                 media_type="text/event-stream",
-                headers={
-                    "Cache-Control": "no-cache",
-                    "Connection": "keep-alive",
-                },
             )
-
-        resp = await _http_client.post(
-            "/v1/chat/completions",
-            json=payload,
-        )
-
-        resp.raise_for_status()
-
-        return JSONResponse(content=resp.json())
-
-    except httpx.HTTPStatusError as e:
-        raise HTTPException(
-            status_code=e.response.status_code,
-            detail=e.response.text,
-        )
-
+        else:
+            response = llm.create_chat_completion(
+                messages=messages_dict,
+                max_tokens=req.max_tokens,
+                temperature=req.temperature,
+                top_p=req.top_p,
+                stream=False,
+            )
+            return JSONResponse(content=response)
+            
     except Exception as e:
-        raise HTTPException(
-            status_code=500,
-            detail=str(e),
-        )
-
+        raise HTTPException(status_code=500, detail=str(e))
 
-# ===========================================================================
-# 8. GRADIO RESPONSE CLEANING
-# ===========================================================================
-
-def clean_assistant_reply(
-    text: Any,
-) -> str:
+# ---------------------------------------------------------------------------
+# ⚡ 5. GRADIO UI & HISTORY CLEANER
+# ---------------------------------------------------------------------------
+def clean_assistant_reply(text):
     if not text:
         return ""
-
     if isinstance(text, list):
-        parts = []
-
-        for p in text:
-            if isinstance(p, str):
-                parts.append(p)
-            elif isinstance(p, dict):
-                parts.append(
-                    p.get(
-                        "text",
-                        p.get("content", ""),
-                    )
-                )
-
-        text = "".join(parts)
-
+        extracted = []
+        for part in text:
+            if isinstance(part, str):
+                extracted.append(part)
+            elif isinstance(part, dict):
+                extracted.append(part.get("text", part.get("content", "")))
+        text = "".join(extracted)
     elif not isinstance(text, str):
         text = str(text)
 
-    cleaned = re.sub(
-        r"<think>.*?</think>\n*",
-        "",
-        text,
-        flags=re.DOTALL,
-    )
-
-    cleaned = cleaned.replace(
-        "</think>",
-        "",
-    )
-
-    cleaned = re.sub(
-        r"\n\n---\n\*⏱️.*?\*$",
-        "",
-        cleaned,
-    )
-
+    cleaned = re.sub(r'<think>.*?</think>\n*', '', text, flags=re.DOTALL)
+    cleaned = cleaned.replace("</think>", "")
+    cleaned = re.sub(r'\n\n---\n\*⏱️.*?\*$', '', cleaned)
     return cleaned.strip()
 
-
-# ===========================================================================
-# 9. GRADIO CHAT FUNCTION
-# ===========================================================================
-
-def respond(
-    message,
-    history,
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
-):
-    messages = [
-        {
-            "role": "system",
-            "content": system_message,
-        }
-    ]
-
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    messages = [{"role": "system", "content": system_message}]
+    
     for item in history:
         if isinstance(item, dict):
             role = item.get("role", "user")
             content = item.get("content", "")
-
             if role == "assistant":
                 content = clean_assistant_reply(content)
-
-            messages.append(
-                {
-                    "role": role,
-                    "content": content,
-                }
-            )
-
+            messages.append({"role": role, "content": content})
         elif isinstance(item, (list, tuple)):
-            if len(item) >= 1 and item[0]:
-                messages.append(
-                    {
-                        "role": "user",
-                        "content": item[0],
-                    }
-                )
-
-            if len(item) >= 2 and item[1]:
-                messages.append(
-                    {
-                        "role": "assistant",
-                        "content": clean_assistant_reply(
-                            item[1]
-                        ),
-                    }
-                )
+            if item[0]:
+                messages.append({"role": "user", "content": item[0]})
+            if item[1]:
+                cleaned_reply = clean_assistant_reply(item[1])
+                messages.append({"role": "assistant", "content": cleaned_reply})
 
-    if isinstance(message, dict):
-        user_text = message.get("text", "")
-
-        if not user_text:
-            user_text = message.get("content", "")
-    else:
-        user_text = str(message)
-
-    messages.append(
-        {
-            "role": "user",
-            "content": user_text,
-        }
-    )
-
-    payload = {
-        "model": MODEL_NAME,
-        "messages": messages,
-        "max_tokens": int(max_tokens),
-        "temperature": float(temperature),
-        "top_p": float(top_p),
-        "stream": True,
-    }
+    messages.append({"role": "user", "content": message})
 
     response = ""
     token_count = 0
     start_time = time.time()
-
-    try:
-        with httpx.stream(
-            "POST",
-            f"{LLAMA_BASE_URL}/v1/chat/completions",
-            json=payload,
-            timeout=None,
-        ) as r:
-            r.raise_for_status()
-
-            for line in r.iter_lines():
-                if not line:
-                    continue
-
-                if not line.startswith("data: "):
-                    continue
-
-                data_str = line[len("data: "):]
-
-                if data_str.strip() == "[DONE]":
-                    break
-
-                try:
-                    chunk = json.loads(data_str)
-                except json.JSONDecodeError:
-                    continue
-
-                choices = chunk.get("choices", [])
-
-                if not choices:
-                    continue
-
-                delta = choices[0].get("delta", {})
-                piece = delta.get("content")
-
-                if piece:
-                    response += piece
-                    token_count += 1
-                    yield response
-
-    except Exception as e:
-        response += f"\n\n⚠️ Generation error: {e}"
-        yield response
-        return
-
-    elapsed = time.time() - start_time
-
-    if elapsed > 0 and token_count > 0:
-        tps = token_count / elapsed
-
-        response += (
-            "\n\n---\n"
-            f"*⏱️ {tps:.2f} tok/s "
-            f"({token_count} tokens in {elapsed:.2f}s) — "
-            f"MTP draft-n-max={SPEC_DRAFT_N_MAX}*"
-        )
-
+    
+    for chunk in llm.create_chat_completion(
+        messages=messages,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        stream=True,
+    ):
+        delta = chunk["choices"][0]["delta"]
+        if "content" in delta and delta["content"]:
+            response += delta["content"]
+            token_count += 1
+            yield response
+            
+    end_time = time.time()
+    elapsed_time = end_time - start_time
+    
+    if elapsed_time > 0 and token_count > 0:
+        tps = token_count / elapsed_time
+        stats_footer = f"\n\n---\n*⏱️ {tps:.2f} tok/s ({token_count} tokens in {elapsed_time:.2f}s)*"
+        response += stats_footer
         yield response
 
-
-# ===========================================================================
-# 10. GRADIO UI
-# ===========================================================================
-
 chatbot = gr.ChatInterface(
     fn=respond,
-
     additional_inputs=[
-        gr.Textbox(
-            value="You are a helpful coding assistant.",
-            label="System message",
-        ),
-
-        gr.Slider(
-            minimum=1,
-            maximum=2048,
-            value=512,
-            step=1,
-            label="Max new tokens",
-        ),
-
-        gr.Slider(
-            minimum=0.1,
-            maximum=2.0,
-            value=0.7,
-            step=0.1,
-            label="Temperature",
-        ),
-
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p",
-        ),
+        gr.Textbox(value="You are a helpful coding assistant.", label="System message"),
+        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
+        gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
     ],
-
-    title=(
-        "Qwen3.6-35B-A3B "
-        "(MTP speculative decoding, CPU)"
-    ),
-)
-
-
-# ===========================================================================
-# 11. MOUNT GRADIO INTO FASTAPI
-# ===========================================================================
-
-app = gr.mount_gradio_app(
-    app,
-    chatbot,
-    path="/",
+    title="FREEAPI RUNNING ON CPU Qwen 35B-A3B",
 )
 
+# Mount Gradio UI inside FastAPI at the root path "/"
+app = gr.mount_gradio_app(app, chatbot, path="/")
 
-# ===========================================================================
-# 12. HF SPACES ENTRYPOINT
-# ===========================================================================
-#
-# Do NOT call uvicorn.run().
-# Do NOT register an atexit handler.
-#
-# Hugging Face Spaces loads the `app` object, and the FastAPI lifespan
-# manages llama-server startup/shutdown.
-# ===========================================================================
+# ---------------------------------------------------------------------------
+# ⚡ 6. SERVER LAUNCH
+# ---------------------------------------------------------------------------
+if __name__ == "__main__":
+    import uvicorn
+    # Port 7860 is required by Hugging Face Spaces for health checks
+    uvicorn.run(app, host="0.0.0.0", port=7860)