Replace Gemini with in-Space open-source LLM (Qwen3.5-2B via llama.cpp)

Dockerfile

@@ -10,6 +10,7 @@ RUN apt-get update && apt-get install -y \
     ffmpeg \
     libsm6 \
     libxext6 \
+    libgomp1 \
     && rm -rf /var/lib/apt/lists/* \
     && git lfs install
 

app.py

@@ -12,6 +12,8 @@ import json
 import os
 import subprocess
 import sys
+import threading
+import time
 from pathlib import Path
 from datetime import datetime
 import logging
@@ -33,13 +35,7 @@ from huggingface_hub import HfApi, hf_hub_download, upload_file
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
-
-# Gemini AI  (new SDK: google-genai)
-try:
-    from google import genai
-    GEMINI_AVAILABLE = True
-except ImportError:
-    GEMINI_AVAILABLE = False
+from starlette.concurrency import run_in_threadpool
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -60,11 +56,22 @@ TRAINING_DATA_DIR = APP_DIR / "training_data"
 TRAINING_DATA_DIR.mkdir(exist_ok=True)
 
 # Environment
-GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY", "")
-GEMINI_MODEL = os.environ.get("GEMINI_MODEL", "gemini-2.5-flash")
 HF_TOKEN = os.environ.get("HF_TOKEN", "")
 HF_MODEL_REPO = os.environ.get("HF_MODEL_REPO", "Che237/cyberforge-models")
 
+# Open-source chat model that runs inside this Space on CPU (llama.cpp, GGUF).
+# It replaces Gemini and needs no API key or credit. Any GGUF chat model can be
+# set here; on the free cpu-basic tier (2 vCPU) stay around 2B parameters or the
+# answers will not fit inside the callers' timeouts.
+LOCAL_LLM_REPO = os.environ.get("LOCAL_LLM_REPO", "unsloth/Qwen3.5-2B-GGUF")
+LOCAL_LLM_FILE = os.environ.get("LOCAL_LLM_FILE", "Qwen3.5-2B-Q4_K_M.gguf")
+LOCAL_LLM_CTX = int(os.environ.get("LOCAL_LLM_CTX", "4096"))
+LOCAL_LLM_THREADS = int(os.environ.get("LOCAL_LLM_THREADS", "2"))
+LOCAL_LLM_MAX_TOKENS = int(os.environ.get("LOCAL_LLM_MAX_TOKENS", "320"))
+LOCAL_LLM_MAX_INPUT_CHARS = int(os.environ.get("LOCAL_LLM_MAX_INPUT_CHARS", "2400"))
+LOCAL_LLM_TIME_BUDGET_S = float(os.environ.get("LOCAL_LLM_TIME_BUDGET_S", "22"))
+LOCAL_LLM_QUEUE_TIMEOUT_S = float(os.environ.get("LOCAL_LLM_QUEUE_TIMEOUT_S", "5"))
+
 logger.info(f"APP_DIR: {APP_DIR}")
 logger.info(f"NOTEBOOKS_DIR: {NOTEBOOKS_DIR}")
 logger.info(f"NOTEBOOKS_DIR exists: {NOTEBOOKS_DIR.exists()}")
@@ -86,131 +93,50 @@ SECURITY_TASKS = [
 
 
 # ============================================================================
-# GEMINI SERVICE  (for REST API)
+# REASONING SERVICE  (LLM-backed analysis for the REST API)
 # ============================================================================
 
-class GeminiService:
-    """Google Gemini AI for cybersecurity chat and analysis"""
-
-    SYSTEM_PROMPT = """You are CyberForge AI, an advanced cybersecurity expert. You specialize in:
-- Real-time threat detection and analysis
-- Malware and phishing identification
-- Network security assessment
-- Browser security monitoring
-- Risk assessment and mitigation
+class ReasoningService:
+    """Structured cybersecurity analysis for the REST endpoints.
 
-When analyzing security queries, provide:
-1. Risk Level (Critical/High/Medium/Low)
-2. Threat Types identified
-3. Confidence Score (0.0-1.0)
-4. Detailed technical analysis
-5. Specific actionable recommendations
+    The text comes from TransformerModelLoader.security_chat: the hosted model
+    when the account has Inference Providers credit, otherwise the open-source
+    model running inside this Space. If no LLM answers, the ML/keyword heuristic
+    in _fallback is used."""
 
-Always be precise, professional, and actionable."""
-
-    def __init__(self):
-        self.client = None
-        self.ready = False
-        self.custom_knowledge = {}
-        self.training_examples = []
-
-    def initialize(self):
-        if not GEMINI_AVAILABLE:
-            logger.warning("google-genai not installed, Gemini unavailable")
-            return
-        if not GEMINI_API_KEY:
-            logger.warning("GEMINI_API_KEY not set, Gemini unavailable")
-            return
-        try:
-            self.client = genai.Client(api_key=GEMINI_API_KEY)
-            resp = self.client.models.generate_content(
-                model=GEMINI_MODEL,
-                contents="Test. Respond with OK."
-            )
-            if resp.text:
-                self.ready = True
-                logger.info(f"✅ Gemini initialized (model: {GEMINI_MODEL})")
-            self._load_knowledge()
-        except Exception as e:
-            logger.error(f"❌ Gemini init failed: {e}")
-            self.ready = False
-
-    def _load_knowledge(self):
-        try:
-            for f in KNOWLEDGE_BASE_DIR.glob("*.json"):
-                with open(f) as fh:
-                    self.custom_knowledge[f.stem] = json.load(fh)
-            logger.info(f"Loaded {len(self.custom_knowledge)} knowledge files")
-        except Exception as e:
-            logger.warning(f"Knowledge load error: {e}")
-        try:
-            for f in TRAINING_DATA_DIR.glob("*.json"):
-                with open(f) as fh:
-                    data = json.load(fh)
-                    if isinstance(data, list):
-                        self.training_examples.extend(data)
-                    else:
-                        self.training_examples.append(data)
-            logger.info(f"Loaded {len(self.training_examples)} training examples")
-        except Exception as e:
-            logger.warning(f"Training data load error: {e}")
-
-    def analyze(self, query: str, context: Dict = None, history: List = None) -> Dict:
-        if not self.ready:
+    def analyze(self, query: str, context: Dict = None, history: List = None,
+                allow_local: bool = True) -> Dict:
+        tl = globals().get("transformer_loader")
+        if tl is None:
             return self._fallback(query)
-        try:
-            knowledge_str = json.dumps(self.custom_knowledge, indent=1)[:2000] if self.custom_knowledge else "None"
-            examples_str = "\n".join(
-                f"Q: {ex.get('input','')}\nA: {ex.get('output','')}"
-                for ex in self.training_examples[-3:]
-            ) if self.training_examples else "None"
-            context_str = f"\nCONTEXT:\n{json.dumps(context, indent=1)[:3000]}\n" if context else ""
-
-            prompt = f"""{self.SYSTEM_PROMPT}
-
-KNOWLEDGE BASE (summary):
-{knowledge_str}
-
-TRAINING EXAMPLES:
-{examples_str}
-{context_str}
-USER QUERY:
-{query}
-
-Provide a comprehensive cybersecurity analysis:"""
-
-            response = self.client.models.generate_content(
-                model=GEMINI_MODEL,
-                contents=prompt,
-                config={"temperature": 0.3, "max_output_tokens": 2048},
-            )
-            text = response.text if response.text else "No response generated"
-            text_lower = text.lower()
-            if "critical" in text_lower:
-                risk_level, risk_score = "Critical", 9.0
-            elif "high" in text_lower:
-                risk_level, risk_score = "High", 7.0
-            elif "medium" in text_lower:
-                risk_level, risk_score = "Medium", 5.0
-            else:
-                risk_level, risk_score = "Low", 3.0
-
-            return {
-                "response": text,
-                "confidence": 0.85,
-                "risk_level": risk_level,
-                "risk_score": risk_score,
-                "insights": [f"Analysis performed by Gemini ({GEMINI_MODEL})"],
-                "recommendations": [],
-                "model_used": GEMINI_MODEL,
-                "timestamp": datetime.utcnow().isoformat(),
-            }
-        except Exception as e:
-            logger.error(f"Gemini analysis error: {e}")
+        ctx = f"\n\nCONTEXT:\n{json.dumps(context, indent=1)[:2000]}" if context else ""
+        llm = tl.security_chat(f"{query}{ctx}", max_tokens=700, allow_local=allow_local)
+        text = llm.get("response")
+        if not text or llm.get("error"):
             return self._fallback(query)
+        tlow = text.lower()
+        if "critical" in tlow:
+            risk_level, risk_score = "Critical", 9.0
+        elif "high" in tlow:
+            risk_level, risk_score = "High", 7.0
+        elif "medium" in tlow:
+            risk_level, risk_score = "Medium", 5.0
+        else:
+            risk_level, risk_score = "Low", 3.0
+        model_id = llm.get("model_id", "llm")
+        return {
+            "response": text,
+            "confidence": 0.82,
+            "risk_level": risk_level,
+            "risk_score": risk_score,
+            "insights": [f"Analysis by {model_id} via {llm.get('source', 'llm')}"],
+            "recommendations": [],
+            "model_used": model_id,
+            "timestamp": datetime.utcnow().isoformat(),
+        }
 
     def _fallback(self, query: str) -> Dict:
-        """ML-powered analysis when Gemini is unavailable"""
+        """ML-powered analysis when no LLM is available"""
         import re
         global ml_loader
 
@@ -313,11 +239,11 @@ Provide a comprehensive cybersecurity analysis:"""
                     "- URL threat analysis (phishing, malware, web attacks)\n"
                     "- Network anomaly detection\n"
                     "- Real-time threat event monitoring\n\n"
-                    "*AI chat (Gemini) is currently unavailable. Provide a URL for full ML-based analysis.*"
+                    "*The AI chat model is currently unavailable. Provide a URL for full ML-based analysis.*"
                 )
             insights = [f"{n_loaded} ML models active"]
 
-        response_lines.append("\n---\n*Powered by CyberForge ML models — Gemini AI offline.*")
+        response_lines.append("\n---\n*Powered by CyberForge ML models — AI chat model offline.*")
 
         return {
             "response": "\n".join(response_lines),
@@ -332,7 +258,7 @@ Provide a comprehensive cybersecurity analysis:"""
 
 
 # Singleton
-gemini_service = GeminiService()
+reasoning_service = ReasoningService()
 
 
 # ============================================================================
@@ -567,8 +493,149 @@ def extract_url_features(url: str) -> Dict:
 #
 # Loads pretrained BERT-based classifiers from the Hugging Face Hub.
 # Models are loaded lazily on first request to keep cold-start fast.
-# A 7B Security LLM is NOT loaded locally (too big for free tier) —
-# it's accessed via the HF Inference API on demand.
+# The chat LLM is separate: see LocalLLM below (runs in this Space) and
+# TransformerModelLoader.security_chat (hosted model first, then LocalLLM).
+
+class LocalLLM:
+    """Open-source chat model served from inside the Space with llama.cpp.
+
+    The GGUF file is downloaded from the Hub once and kept in memory. Generation
+    is serialized (a llama.cpp context is not thread-safe) and time-boxed, so a
+    slow CPU cannot hold a request past the callers' timeouts."""
+
+    def __init__(self):
+        self.llm = None
+        self.error: Optional[str] = None
+        self._chatml = False
+        self._prefill = ""
+        self._load_lock = threading.Lock()
+        self._gen_lock = threading.Lock()
+
+    @property
+    def ready(self) -> bool:
+        return self.llm is not None
+
+    def status(self) -> Dict:
+        return {
+            "repo": LOCAL_LLM_REPO,
+            "file": LOCAL_LLM_FILE,
+            "loaded": self.ready,
+            "loading": self._load_lock.locked(),
+            "error": self.error,
+        }
+
+    def load(self):
+        """Download and load the model. Safe to call repeatedly."""
+        with self._load_lock:
+            if self.llm is not None or self.error:
+                return self.llm
+            try:
+                from llama_cpp import Llama
+                try:
+                    path = hf_hub_download(repo_id=LOCAL_LLM_REPO, filename=LOCAL_LLM_FILE, token=HF_TOKEN or None)
+                except Exception:
+                    # A stale HF_TOKEN must not block the download of a public model
+                    path = hf_hub_download(repo_id=LOCAL_LLM_REPO, filename=LOCAL_LLM_FILE, token=False)
+                self._probe(path)
+                llm = Llama(model_path=path, n_ctx=LOCAL_LLM_CTX, n_threads=LOCAL_LLM_THREADS, verbose=False)
+                template = str((llm.metadata or {}).get("tokenizer.chat_template", ""))
+                self._chatml = "<|im_start|>" in template
+                # Qwen3.5 answers directly when the assistant turn opens with an
+                # empty think block; this is what its template emits for
+                # enable_thinking=False.
+                self._prefill = "<think>\n\n</think>\n\n" if "<think>" in template else ""
+                self.llm = llm
+                logger.info(f"✅ Local LLM loaded: {LOCAL_LLM_REPO}/{LOCAL_LLM_FILE}")
+            except Exception as e:
+                self.error = f"{type(e).__name__}: {str(e)[:300]}"
+                logger.error(f"❌ Local LLM unavailable: {self.error}")
+            return self.llm
+
+    @staticmethod
+    def _probe(path: str) -> None:
+        """Load the model once in a child process. A native crash there (for
+        example a CPU instruction the prebuilt wheel needs but this machine
+        lacks) raises here instead of killing the API process."""
+        code = (
+            "import sys; from llama_cpp import Llama; "
+            "m = Llama(model_path=sys.argv[1], n_ctx=256, n_threads=1, verbose=False); "
+            "m.create_completion('Hi', max_tokens=1)"
+        )
+        r = subprocess.run([sys.executable, "-c", code, path], capture_output=True, text=True, timeout=300)
+        if r.returncode != 0:
+            raise RuntimeError(f"llama.cpp probe exited with {r.returncode}: {(r.stderr or '')[-200:]}")
+
+    def chat(self, system: str, user: str, max_tokens: int = 512) -> Dict:
+        base = {"model": "local-llm", "source": "space_local_llamacpp", "model_id": LOCAL_LLM_REPO}
+        if self.llm is None:
+            if self._load_lock.locked():
+                return {**base, "response": None, "error": "local LLM is still loading"}
+            self.load()
+        if self.llm is None:
+            return {**base, "response": None, "error": self.error or "local LLM not loaded"}
+        if not self._gen_lock.acquire(timeout=LOCAL_LLM_QUEUE_TIMEOUT_S):
+            return {**base, "response": None, "error": "local LLM is busy with another request"}
+        started = time.monotonic()
+        stream = None
+        try:
+            max_tokens = max(16, min(int(max_tokens), LOCAL_LLM_MAX_TOKENS))
+            # The chat markers are structural: drop them from untrusted text
+            system = system.replace("<|im_start|>", "").replace("<|im_end|>", "")
+            user = user.replace("<|im_start|>", "").replace("<|im_end|>", "")
+            if len(user) > LOCAL_LLM_MAX_INPUT_CHARS:
+                # Keep both ends: context sits at the start, the question at the end
+                half = LOCAL_LLM_MAX_INPUT_CHARS // 2
+                user = f"{user[:half]}\n[...]\n{user[-half:]}"
+            if self._chatml:
+                prompt = (
+                    f"<|im_start|>system\n{system}<|im_end|>\n"
+                    f"<|im_start|>user\n{user}<|im_end|>\n"
+                    f"<|im_start|>assistant\n{self._prefill}"
+                )
+                stream = self.llm.create_completion(
+                    prompt, max_tokens=max_tokens, temperature=0.7, top_p=0.8, top_k=20,
+                    presence_penalty=1.5, stop=["<|im_end|>", "<|endoftext|>"], stream=True,
+                )
+                read = lambda c: c["choices"][0].get("text") or ""
+            else:
+                stream = self.llm.create_chat_completion(
+                    messages=[{"role": "system", "content": system}, {"role": "user", "content": user}],
+                    max_tokens=max_tokens, temperature=0.7, top_p=0.8, stream=True,
+                )
+                read = lambda c: c["choices"][0].get("delta", {}).get("content") or ""
+            pieces: List[str] = []
+            truncated = False
+            for chunk in stream:
+                pieces.append(read(chunk))
+                if time.monotonic() - started > LOCAL_LLM_TIME_BUDGET_S:
+                    truncated = True
+                    break
+            import re
+            text = re.sub(r"<think>.*?</think>", "", "".join(pieces), flags=re.S).strip()
+            if truncated:
+                # Out of time: end on the last complete sentence or line
+                cut = max(text.rfind(". "), text.rfind(".\n"), text.rfind("\n"))
+                if cut > len(text) * 0.5:
+                    text = text[:cut + 1].strip()
+            if not text:
+                return {**base, "response": None, "error": "local LLM returned no text"}
+            return {
+                **base,
+                "response": text,
+                "tokens": len(pieces),
+                "elapsed_ms": round((time.monotonic() - started) * 1000),
+                "truncated": truncated,
+            }
+        except Exception as e:
+            return {**base, "response": None, "error": f"{type(e).__name__}: {str(e)[:200]}"}
+        finally:
+            if stream is not None and hasattr(stream, "close"):
+                stream.close()
+            self._gen_lock.release()
+
+
+local_llm = LocalLLM()
+
 
 class TransformerModelLoader:
     """Loads pretrained Transformer classifiers from the HF Hub on demand."""
@@ -585,14 +652,17 @@ class TransformerModelLoader:
             "desc":  "BERT-based URL phishing/malware classifier",
         },
     }
-    # Model used via HF Inference API. ZySec-AI/SecurityLLM isn't served on the
-    # free Inference API tier (404). Mistral-7B-Instruct-v0.3 is widely available
-    # and works well for cyber Q&A. Override via SECURITY_LLM_MODEL env var.
-    SECURITY_LLM_REPO = os.environ.get("SECURITY_LLM_MODEL", "mistralai/Mistral-7B-Instruct-v0.3")
+    # Hosted reasoning model through HF Inference Providers (OpenAI-compatible
+    # router at router.huggingface.co/v1). Billed per token, so it only answers
+    # while the account has credit; otherwise security_chat uses LocalLLM.
+    # Override via REASONING_LLM_MODEL.
+    REASONING_LLM_REPO = os.environ.get("REASONING_LLM_MODEL", "deepseek-ai/DeepSeek-V3-0324")
 
     def __init__(self):
         self.pipelines = {}     # name → transformers.Pipeline (lazy-loaded)
         self.load_errors = {}   # name → last error message
+        self.last_engine = None        # "hosted" | "local": which LLM answered last
+        self._provider_retry_at = 0.0  # monotonic time before which the hosted model is skipped
         self.transformers_available = False
         try:
             import transformers  # noqa: F401
@@ -694,53 +764,83 @@ class TransformerModelLoader:
             "inference_source": "entropy-heuristic",
         }
 
-    def security_chat(self, query: str, max_tokens: int = 512) -> Dict:
-        """Cybersecurity Q&A via ZySec-AI/SecurityLLM hosted on HF Inference API.
-        Falls back to Gemini when the LLM is rate-limited or HF token is missing."""
-        if not HF_TOKEN:
-            return {
-                "model": "security-llm",
-                "response": None,
-                "error": "HF_TOKEN env var required to call ZySec-AI/SecurityLLM via Inference API",
-                "fallback_available": gemini_service.ready,
-            }
+    SYSTEM_PROMPT = (
+        "You are CyberForge AI, a precise, professional cybersecurity analyst. "
+        "Give concise, technical, actionable answers. State a Risk Level "
+        "(Critical/High/Medium/Low) when assessing a threat, and cite CVE / MITRE "
+        "ATT&CK IDs where relevant."
+    )
+
+    def provider_available(self) -> bool:
+        """True when the hosted model is worth trying: a token is set and the
+        last call did not fail on billing or auth."""
+        return bool(HF_TOKEN) and time.monotonic() >= self._provider_retry_at
+
+    def _provider_chat(self, query: str, max_tokens: int) -> Optional[Dict]:
+        """Ask the hosted model. Returns None when it cannot answer."""
+        import requests
         try:
-            import requests
-            url = f"https://api-inference.huggingface.co/models/{self.SECURITY_LLM_REPO}"
-            headers = {"Authorization": f"Bearer {HF_TOKEN}", "Content-Type": "application/json"}
-            payload = {
-                "inputs": query[:2000],
-                "parameters": {"max_new_tokens": max_tokens, "temperature": 0.3, "return_full_text": False},
-                "options": {"wait_for_model": True},
-            }
-            r = requests.post(url, headers=headers, json=payload, timeout=45)
+            r = requests.post(
+                "https://router.huggingface.co/v1/chat/completions",
+                headers={"Authorization": f"Bearer {HF_TOKEN}", "Content-Type": "application/json"},
+                json={
+                    "model": self.REASONING_LLM_REPO,
+                    "messages": [
+                        {"role": "system", "content": self.SYSTEM_PROMPT},
+                        {"role": "user", "content": query[:6000]},
+                    ],
+                    "max_tokens": max_tokens,
+                    "temperature": 0.3,
+                },
+                timeout=60,
+            )
             if r.status_code == 200:
                 data = r.json()
-                text = data[0].get("generated_text") if isinstance(data, list) and data else (data.get("generated_text") if isinstance(data, dict) else str(data))
+                text = data["choices"][0]["message"]["content"]
                 return {
-                    "model":    "security-llm",
-                    "source":   "huggingface_inference_api",
+                    "model":    "deepseek",
+                    "source":   "hf_inference_providers",
                     "response": text,
-                    "model_id": self.SECURITY_LLM_REPO,
+                    "model_id": self.REASONING_LLM_REPO,
                 }
-            return {
-                "model":  "security-llm",
-                "error":  f"HF Inference API HTTP {r.status_code}: {r.text[:200]}",
-                "fallback_available": gemini_service.ready,
-            }
+            # 401/402/403 (bad token, no credit) will not fix themselves quickly
+            self._provider_retry_at = time.monotonic() + (900 if r.status_code in (401, 402, 403) else 60)
+            logger.warning(f"Hosted LLM HTTP {r.status_code}: {r.text[:160]} — using the local model")
         except Exception as e:
-            return {
-                "model":  "security-llm",
-                "error":  f"{type(e).__name__}: {str(e)[:200]}",
-                "fallback_available": gemini_service.ready,
-            }
+            self._provider_retry_at = time.monotonic() + 60
+            logger.warning(f"Hosted LLM error: {type(e).__name__}: {str(e)[:160]} — using the local model")
+        return None
+
+    def security_chat(self, query: str, max_tokens: int = 512, allow_local: bool = True) -> Dict:
+        """Cybersecurity Q&A.
+        1. Hosted model through HF Inference Providers, while the account has credit.
+        2. The open-source model running inside this Space (free, no key)."""
+        if self.provider_available():
+            hosted = self._provider_chat(query, max_tokens)
+            if hosted:
+                self.last_engine = "hosted"
+                return hosted
+        if not allow_local:
+            return {"model": "security-llm", "response": None, "error": "hosted LLM unavailable"}
+        result = local_llm.chat(self.SYSTEM_PROMPT, query, max_tokens=max_tokens)
+        if result.get("response"):
+            self.last_engine = "local"
+        return result
 
     def status(self) -> Dict:
         return {
             "transformers_available": self.transformers_available,
             "loaded": list(self.pipelines.keys()),
-            "available": list(self.REGISTRY.keys()) + ["security_llm (via HF Inference API)"],
+            "available": list(self.REGISTRY.keys()) + ["reasoning_llm"],
             "load_errors": self.load_errors,
+            "reasoning_llm": {
+                "last_engine": self.last_engine,
+                "local": local_llm.status(),
+                "hosted": {
+                    "model": self.REASONING_LLM_REPO,
+                    "usable": self.provider_available(),
+                },
+            },
         }
 
     @staticmethod
@@ -1027,7 +1127,9 @@ async def api_health():
         "status": "healthy",
         "timestamp": datetime.utcnow().isoformat(),
         "services": {
-            "gemini": gemini_service.ready,
+            "reasoning_llm": transformer_loader.last_engine,  # "hosted" | "local" | None until first answer
+            "reasoning_model": transformer_loader.REASONING_LLM_REPO if HF_TOKEN else None,
+            "local_llm": local_llm.status(),
             "ml_models": ml_loader.ready,
             "models_loaded": list(ml_loader.models.keys()),
             "transformers": transformer_loader.transformers_available,
@@ -1053,7 +1155,7 @@ async def api_analyze(request: Request):
     query = body.get("query", "")
     context = body.get("context", {})
     history = body.get("conversation_history", [])
-    result = gemini_service.analyze(query, context, history)
+    result = await run_in_threadpool(reasoning_service.analyze, query, context, history)
     return result
 
 
@@ -1131,7 +1233,7 @@ async def api_scan_threats(request: Request):
     """Threat scanning – called by backend mlService.scanForThreats()"""
     body = await request.json()
     query = json.dumps(body.get("data", body), indent=1)[:3000]
-    result = gemini_service.analyze(f"Scan these indicators for threats:\n{query}")
+    result = await run_in_threadpool(reasoning_service.analyze, f"Scan these indicators for threats:\n{query}")
     return result
 
 
@@ -1141,7 +1243,7 @@ async def api_generate_insights(request: Request):
     body = await request.json()
     query = body.get("query", "")
     context = body.get("context", {})
-    result = gemini_service.analyze(query, context)
+    result = await run_in_threadpool(reasoning_service.analyze, query, context)
     return {
         "insights": result.get("response", ""),
         "confidence": result.get("confidence", 0),
@@ -1200,20 +1302,18 @@ async def api_v2_dga_detect(request: Request):
 
 @api.post("/api/v2/security-chat")
 async def api_v2_security_chat(request: Request):
-    """Cybersecurity Q&A via ZySec-AI/SecurityLLM (HF Inference API).
+    """Cybersecurity Q&A: hosted model when the account has credit, otherwise
+    the open-source model running in this Space.
     Body: { "query": "...", "max_tokens": 512 }"""
     body = await request.json()
     query = body.get("query", "").strip()
     if not query:
         return JSONResponse(status_code=400, content={"detail": "query required"})
     max_tokens = int(body.get("max_tokens", 512))
-    result = transformer_loader.security_chat(query, max_tokens=max_tokens)
-    # Auto-fallback to Gemini when LLM unavailable
-    if result.get("error") and gemini_service.ready:
-        gemini_result = gemini_service.analyze(query)
-        gemini_result["source"] = "gemini-fallback"
-        gemini_result["llm_error"] = result["error"]
-        return gemini_result
+    result = await run_in_threadpool(transformer_loader.security_chat, query, max_tokens)
+    # Last resort: ML heuristic if no LLM answered
+    if result.get("error"):
+        return reasoning_service._fallback(query)
     return result
 
 
@@ -1324,7 +1424,7 @@ async def api_v2_ioc_scan(request: Request):
         "domain": "example.com",           # optional
         "ip":     "1.2.3.4",              # optional
         "hash":   "abc123...",             # optional (future use)
-        "context": { ... }                 # optional — passed to Gemini if available
+        "context": { ... }                 # optional — passed to the LLM narrative
       }
 
     Response: per-indicator results with unified risk summary.
@@ -1382,13 +1482,15 @@ async def api_v2_ioc_scan(request: Request):
     else:
         risk_level = "unknown"
 
-    # Optional Gemini enrichment when available
+    # Optional LLM narrative. Hosted model only: the in-Space model takes many
+    # seconds on CPU and this endpoint is called with short timeouts.
     ai_analysis = None
-    if gemini_service.ready and (url or domain):
+    if transformer_loader.provider_available() and (url or domain):
         target = url or domain
-        ai_analysis = gemini_service.analyze(
+        ai_analysis = await run_in_threadpool(
+            reasoning_service.analyze,
             f"Scan these IOC indicators for threats: url={target}, domain={effective_domain}",
-            context=context,
+            context, None, False,
         )
 
     return {
@@ -1506,8 +1608,9 @@ async def api_v2_chat(request: Request):
          persistent vector store, which requires paid GPU on free tier).
       2. Translates raw ML signals / IOCs into plain human language when the
          user includes raw data and asks for an explanation.
-      3. Falls back to the Security LLM (Mistral-7B via HF Inference API)
-         or the ML-powered heuristic analyzer when Gemini is offline.
+      3. Uses the hosted model when the account has credit, otherwise the
+         open-source model running in this Space,
+         or the ML-powered heuristic analyzer when no LLM is available.
 
     NOTE ON REAL-TIME "LEARNING": HuggingFace Spaces cpu-basic does NOT
     support fine-tuning or persistent model updates at runtime. What this
@@ -1674,22 +1777,9 @@ async def api_v2_chat(request: Request):
         n_models = len(ml_loader.models)
         sources.append({"type": "ml_models", "label": "ML Models", "value": f"{n_models}/4 loaded"})
 
-    # ── LLM cascade: Gemini → Security LLM → ML heuristic ────────────
-    # Path 1: Gemini (premium, requires GEMINI_API_KEY env var)
-    if gemini_service.ready:
-        result = gemini_service.analyze(enriched_message, context=context, history=history)
-        # Merge detected risk if ML gave a stronger signal
-        if detected_risk_score > 0 and detected_risk_level != "unknown":
-            ml_score_100 = round(detected_risk_score * 100, 1)
-            if ml_score_100 > result.get("risk_score", 0):
-                result["risk_score"] = ml_score_100
-                result["risk_level"] = detected_risk_level
-        result["session_id"] = session_id
-        result["sources"] = sources
-        return result
-
-    # Path 2: Security LLM via HF Inference API (Mistral-7B)
-    llm_result = transformer_loader.security_chat(enriched_message, max_tokens=600)
+    # LLM cascade: hosted model (when the account has credit), then the
+    # open-source model in this Space, then the ML heuristic below.
+    llm_result = await run_in_threadpool(transformer_loader.security_chat, enriched_message, 600)
     if llm_result.get("response") and not llm_result.get("error"):
         text = llm_result["response"]
         text_lower = text.lower()
@@ -1712,14 +1802,14 @@ async def api_v2_chat(request: Request):
             "confidence": 0.72,
             "risk_level": risk_level,
             "risk_score": risk_score,
-            "model_used": llm_result.get("model_id", transformer_loader.SECURITY_LLM_REPO),
+            "model_used": llm_result.get("model_id", LOCAL_LLM_REPO),
             "session_id": session_id,
             "sources": sources,
             "timestamp": datetime.utcnow().isoformat(),
         }
 
     # Path 3: ML heuristic fallback (always available)
-    fallback = gemini_service._fallback(enriched_message)
+    fallback = reasoning_service._fallback(enriched_message)
     # Override with ML scan risk if available
     if detected_risk_score > 0:
         ml_score_100 = round(detected_risk_score * 100, 1)
@@ -1736,8 +1826,9 @@ async def api_analyze_network(request: Request):
     """Network traffic analysis – called by backend mlService.analyzeNetworkTraffic()"""
     body = await request.json()
     traffic_data = body.get("traffic_data", {})
-    result = gemini_service.analyze(
-        f"Analyze this network traffic for security threats:\n{json.dumps(traffic_data, indent=1)[:3000]}"
+    result = await run_in_threadpool(
+        reasoning_service.analyze,
+        f"Analyze this network traffic for security threats:\n{json.dumps(traffic_data, indent=1)[:3000]}",
     )
     return result
 
@@ -1748,8 +1839,9 @@ async def api_execute_task(request: Request):
     body = await request.json()
     task_type = body.get("task_type", "")
     task_data = body.get("task_data", {})
-    result = gemini_service.analyze(
-        f"Execute cybersecurity task '{task_type}':\n{json.dumps(task_data, indent=1)[:3000]}"
+    result = await run_in_threadpool(
+        reasoning_service.analyze,
+        f"Execute cybersecurity task '{task_type}':\n{json.dumps(task_data, indent=1)[:3000]}",
     )
     return result
 
@@ -1759,8 +1851,9 @@ async def api_analyze_browser(request: Request):
     """Browser session analysis"""
     body = await request.json()
     session = body.get("session_data", body)
-    result = gemini_service.analyze(
-        f"Analyze this browser session for security threats:\n{json.dumps(session, indent=1)[:3000]}"
+    result = await run_in_threadpool(
+        reasoning_service.analyze,
+        f"Analyze this browser session for security threats:\n{json.dumps(session, indent=1)[:3000]}",
     )
     return result
 
@@ -1769,8 +1862,9 @@ async def api_analyze_browser(request: Request):
 async def api_analyze_threat_feed(request: Request):
     """Threat feed analysis"""
     body = await request.json()
-    result = gemini_service.analyze(
-        f"Analyze this threat feed:\n{json.dumps(body, indent=1)[:3000]}"
+    result = await run_in_threadpool(
+        reasoning_service.analyze,
+        f"Analyze this threat feed:\n{json.dumps(body, indent=1)[:3000]}",
     )
     return result
 
@@ -1871,9 +1965,9 @@ def create_interface():
 | Method | Endpoint | Description |
 |--------|----------|-------------|
 | GET | `/health` | Health check — version, models, services |
-| POST | `/analyze` | AI chat / general analysis (Gemini or ML fallback) |
+| POST | `/analyze` | AI chat / general analysis (LLM or ML fallback) |
 | POST | `/analyze-url` | URL threat analysis — 4 ML models + risk factors |
-| POST | `/scan-threats` | Deep threat scan (Gemini or ML fallback) |
+| POST | `/scan-threats` | Deep threat scan (LLM or ML fallback) |
 | POST | `/api/insights/generate` | AI-generated security insights |
 | POST | `/api/models/predict` | Direct ML model prediction |
 | GET | `/models` | List loaded models with source info |
@@ -1889,7 +1983,7 @@ def create_interface():
 | POST | `/api/v2/ioc-scan` | Multi-IOC scanner: URL + domain + DGA + BERT |
 | POST | `/api/v2/url-enrich` | Enriched analysis: ML + BERT + DGA + explainability |
 | POST | `/api/v2/chat` | Security chatbot — context-grounded, RAG-lite, machine→human translation |
-| POST | `/api/v2/security-chat` | Security LLM Q&A (Mistral-7B via HF Inference API) |
+| POST | `/api/v2/security-chat` | Security LLM Q&A (hosted model, or the open-source model in this Space) |
 | GET | `/api/v2/status` | Transformer model load status |
 
 ### Quick Examples
@@ -1931,8 +2025,9 @@ curl -X POST https://che237-cyberforge.hf.space/api/v2/chat \\
 
 # Initialize services at startup
 logger.info("🚀 Initializing CyberForge AI services...")
-gemini_service.initialize()
 ml_loader.initialize()
+# Load the chat model in the background so the API answers health checks at once
+threading.Thread(target=local_llm.load, name="local-llm-loader", daemon=True).start()
 logger.info("✅ Services initialized")
 
 # Create Gradio app and mount it on FastAPI

requirements.txt

@@ -19,14 +19,20 @@ gradio>=5.20.0
 fastapi>=0.100.0
 uvicorn[standard]>=0.22.0
 
-# HF Hub
-huggingface_hub>=0.25.0
+# HF Hub (>=0.26 for Inference Providers / OpenAI-compatible router used by DeepSeek)
+huggingface_hub>=0.26.0
 
 # Feature Engineering
 tldextract>=5.1.0
 
-# Gemini AI (new SDK)
-google-genai>=1.0.0
+# Reasoning LLM
+#   hosted: HF Inference Providers (OpenAI-compatible router), plain HTTPS via requests.
+#   local:  open-source GGUF chat model run in-process with llama.cpp. The prebuilt
+#           CPU wheel is used so the image needs no compiler; --only-binary makes the
+#           build fail fast instead of compiling from source if that wheel is missing.
+--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+--only-binary llama-cpp-python
+llama-cpp-python==0.3.36
 
 # Notebook execution
 nbformat>=5.10.0