Update app.py

app.py

@@ -1,105 +1,124 @@
 import os
 import sys
+import time
+import re
 import multiprocessing
 import subprocess
 
 # ---------------------------------------------------------------------------
-# ⚡ 1. OS & CPU INSTRUCTION TUNING (Must be set BEFORE importing llama_cpp)
+# ⚡ 1. INSTALL & ENVIRONMENT TUNING
 # ---------------------------------------------------------------------------
-physical_cores = max(1, multiprocessing.cpu_count() // 2)
-
-# Lock math libraries strictly to physical CPU cores
-os.environ["OMP_NUM_THREADS"] = str(physical_cores)
-os.environ["MKL_NUM_THREADS"] = str(physical_cores)
-os.environ["OPENBLAS_NUM_THREADS"] = str(physical_cores)
-os.environ["VECLIB_MAXIMUM_THREADS"] = str(physical_cores)
+# Auto-detect physical CPU cores to avoid hyperthreading contention
+try:
+    import psutil
+    PHYSICAL_CORES = psutil.cpu_count(logical=False) or max(1, multiprocessing.cpu_count() // 2)
+except Exception:
+    PHYSICAL_CORES = max(1, multiprocessing.cpu_count() // 2)
 
-# Core Pinning: Prevents cross-core cache invalidation & context switches
+# Lock thread pools strictly to physical CPU cores
+os.environ["OMP_NUM_THREADS"] = str(PHYSICAL_CORES)
+os.environ["MKL_NUM_THREADS"] = str(PHYSICAL_CORES)
+os.environ["OPENBLAS_NUM_THREADS"] = str(PHYSICAL_CORES)
 os.environ["OMP_PROC_BIND"] = "TRUE"
-os.environ["OMP_PLACES"] = "cores"
 
-# Install prebuilt CPU wheel and psutil for process priority
+# Install prebuilt CPU wheel and dependencies at runtime
 subprocess.run([
     sys.executable, "-m", "pip", "install",
-    "llama-cpp-python", "psutil",
+    "llama-cpp-python", "psutil", "gradio", "huggingface_hub",
     "--extra-index-url", "https://abetlen.github.io/llama-cpp-python/whl/cpu",
 ], check=True)
 
-import time
-import re
-import psutil
 import gradio as gr
 import llama_cpp
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
-# ---------------------------------------------------------------------------
-# ⚡ 2. HIGH OS SCHEDULER PRIORITY
-# ---------------------------------------------------------------------------
+# Raise OS scheduler priority for the current process
 try:
     p = psutil.Process()
     if sys.platform == "win32":
         p.nice(psutil.HIGH_PRIORITY_CLASS)
     else:
-        p.nice(-15)  # Highest CPU scheduling priority on Linux/Mac
+        p.nice(-15)
 except Exception:
     pass
 
-NUMA_STRATEGY = getattr(llama_cpp, "GGML_NUMA_STRATEGY_DISTRIBUTE", True)
+# ---------------------------------------------------------------------------
+# ⚡ 2. MODEL DOWNLOAD & LLAMA-CPP INITIALIZATION
+# ---------------------------------------------------------------------------
+NUMA_STRATEGY = getattr(llama_cpp, "GGML_NUMA_STRATEGY_DISTRIBUTE", False)
 
-# Download GGUF Model
 model_path = hf_hub_download(
     "unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF",
     "Qwen3-Coder-30B-A3B-Instruct-UD-IQ3_XXS.gguf",
 )
 
-# ---------------------------------------------------------------------------
-# ⚡ 3. MAXIMIZED CPU ENGINE CONFIGURATION
-# ---------------------------------------------------------------------------
+# Load GGUF model optimized for CPU & MoE execution
 llm = Llama(
     model_path=model_path,
-    n_gpu_layers=0,      
+    n_gpu_layers=0,             # 0 for CPU execution
     n_ctx=4096,
-    chat_format="chatml",
+    chat_format="chatml",       # ChatML format prevents KV cache issues
     
-    # Core Pinning & Scheduling
-    n_threads=physical_cores,                           # Pure physical cores for text generation
-    n_threads_batch=multiprocessing.cpu_count(),       # Burst ALL logical cores for prompt evaluation
+    # Threading optimizations
+    n_threads=PHYSICAL_CORES,       
+    n_threads_batch=PHYSICAL_CORES, 
     numa=NUMA_STRATEGY,
     
-    # RAM Pinning & Memory Bandwidth Maxing
-    use_mlock=True,    # Force model weights locked in physical RAM (no swap/disk paging)
-    use_mmap=True,     # High-speed direct memory mapping
+    # Memory & Cache Management
+    use_mlock=False,            # Set True if running locally with spare RAM
+    use_mmap=True,
+    flash_attn=False,           # Disabled on CPU to reduce attention computation overhead
+    type_k=llama_cpp.GGML_TYPE_Q4_0,  # Q4 KV cache cuts RAM consumption in half
+    type_v=llama_cpp.GGML_TYPE_Q4_0,
     
-    # Quantized KV Cache & Flash Attention
-    flash_attn=True,
-    type_k=llama_cpp.GGML_TYPE_Q4_0,    
-    type_v=llama_cpp.GGML_TYPE_Q4_0,    
-    
-    # Batch Memory Alignment
-    n_batch=512,    
-    n_ubatch=512,   
+    # Batch Memory Alignment (ubatch=64 prevents CPU L3 cache eviction)
+    n_batch=256,    
+    n_ubatch=64,   
 )
 
-# Clean thoughts and performance metadata from history
+# ---------------------------------------------------------------------------
+# ⚡ 3. ROBUST HISTORY CLEANER (FIXES GRADIO 5 TYPEERROR & THINK LOOPS)
+# ---------------------------------------------------------------------------
 def clean_assistant_reply(text):
     if not text:
-        return text
+        return ""
+    
+    # Safely handle Gradio 5 / Multimodal list & dict formats
+    if isinstance(text, list):
+        extracted = []
+        for part in text:
+            if isinstance(part, str):
+                extracted.append(part)
+            elif isinstance(part, dict):
+                extracted.append(part.get("text", part.get("content", "")))
+        text = "".join(extracted)
+    elif not isinstance(text, str):
+        text = str(text)
+
+    # 1. Strip reasoning blocks (<think>...</think>)
     cleaned = re.sub(r'<think>.*?</think>\n*', '', text, flags=re.DOTALL)
-    cleaned = cleaned.replace('</think>', '')
+    cleaned = cleaned.replace("</think>", "")
+    
+    # 2. Strip previous timing stats footers
     cleaned = re.sub(r'\n\n---\n\*⏱️.*?\*$', '', cleaned)
+    
     return cleaned.strip()
 
+# ---------------------------------------------------------------------------
+# ⚡ 4. GRADIO RESPOND FUNCTION (WITH TOK/S TRACKING)
+# ---------------------------------------------------------------------------
 def respond(message, history, system_message, max_tokens, temperature, top_p):
     messages = [{"role": "system", "content": system_message}]
     
+    # Parse chat history safely
     for item in history:
         if isinstance(item, dict):
+            role = item.get("role", "user")
             content = item.get("content", "")
-            if item.get("role") == "assistant":
+            if role == "assistant":
                 content = clean_assistant_reply(content)
-            messages.append({"role": item.get("role"), "content": content})
-            
+            messages.append({"role": role, "content": content})
         elif isinstance(item, (list, tuple)):
             if item[0]:
                 messages.append({"role": "user", "content": item[0]})
@@ -126,6 +145,7 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
             token_count += 1
             yield response
             
+    # Calculate generation speed
     end_time = time.time()
     elapsed_time = end_time - start_time
     
@@ -135,6 +155,9 @@ def respond(message, history, system_message, max_tokens, temperature, top_p):
         response += stats_footer
         yield response
 
+# ---------------------------------------------------------------------------
+# ⚡ 5. GRADIO INTERFACE LAUNCH
+# ---------------------------------------------------------------------------
 chatbot = gr.ChatInterface(
     fn=respond,
     additional_inputs=[
@@ -143,9 +166,8 @@ chatbot = gr.ChatInterface(
         gr.Slider(minimum=0.1, maximum=2.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p"),
     ],
-    title="Qwopus 3.5 Coder (Max-Performance CPU Edition)",
+    title="Qwopus 3.5 Coder (Ultra-Optimized CPU/MoE Edition)",
 )
 
 if __name__ == "__main__":
-    chatbot.launch()
-    
+    chatbot.launch()