backend.py

"""
Claude Code Backend — Agentic coding backend powered by NVIDIA NIM models.
Exposes an OpenAI-compatible /v1/chat/completions endpoint with built-in
tools for file operations and bash execution.

Architecture:
  Space 1 (better-chatbot) --> this backend --> NVIDIA NIM API
  
The agentic loop:
  1. Receive user message from Space 1
  2. Send to NIM model with tool definitions
  3. If model returns tool_calls, execute them and loop
  4. If model returns text, stream it back to Space 1
  5. Persist conversation in Postgres
"""

import os
import shutil
import threading
import json
import uuid
import subprocess
import asyncio
import time
import re
import collections
from pathlib import Path
from typing import AsyncIterator, Optional, List, Dict, Any
from pydantic import BaseModel
import contextvars
import urllib.parse

workspace_var = contextvars.ContextVar("workspace_dir", default="/tmp/workspace")

# --- Ultimate Agent Brain Imports ---
from second_brain import SecondBrainWrapper
from survival_watchdog import SurvivalWatchdog, get_metrics
from swarm_llm import swarm
from helix_state import helix_db
from context_engine import ContextEngine

# Instantiate singletons for the orchestrator
SPACE_NAME = os.environ.get("SPACE_NAME", "space2-cerebrum")
brain = SecondBrainWrapper(space_name=SPACE_NAME)
context_engine = ContextEngine(brain)
watchdog = SurvivalWatchdog()


from fastapi import FastAPI, Request, Header, HTTPException
from fastapi.responses import StreamingResponse, JSONResponse, HTMLResponse
from fastapi.middleware.cors import CORSMiddleware
from openai import AsyncOpenAI
import anyio
import asyncpg

# ---------------------------------------------------------------------------
# Globals & Activity Logs
# ---------------------------------------------------------------------------
activity_logs = collections.deque(maxlen=100)
MODEL_STATUSES = {}
ACTIVE_SESSIONS = set()

def log_activity(msg: str):
    timestamp = time.strftime("%H:%M:%S")
    log_line = f"[{timestamp}] {msg}"
    activity_logs.append(log_line)
    print(log_line)

# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------

NIM_API_KEY = os.environ.get("NVIDIA_NIM_API_KEY", "")
BACKEND_API_KEY = os.environ.get("BACKEND_API_KEY", "")
DATABASE_URL = os.environ.get("DATABASE_URL", "")
WORKSPACE_DIR = os.environ.get("WORKSPACE_DIR", "/tmp/workspace")
BACKUP_GIT_REPO = os.environ.get("BACKUP_GIT_REPO", "")
MAX_TOOL_ROUNDS = int(os.environ.get("MAX_TOOL_ROUNDS", "10"))

# NIM models that reliably support tool/function calling
TOOL_CAPABLE_MODELS = {
    "nvidia/nemotron-3-ultra-550b-a55b": "Nemotron 3 Ultra 550B (Agentic)",
    "z-ai/glm-5.1": "GLM 5.1 (Agentic)",
    "moonshotai/kimi-k2.6": "Kimi K2.6 (Agentic)",
    "minimaxai/minimax-m3": "MiniMax M3 (Agentic)",
    "stepfun-ai/step-3.7-flash": "Step 3.7 Flash (Agentic)",
    "minimaxai/minimax-m2.7": "MiniMax M2.7 (Agentic)",
    "meta/llama-3.1-70b-instruct": "Llama 3.1 70B (Agentic)",
    "meta/llama-3.1-405b-instruct": "Llama 3.1 405B (Agentic)",
    "qwen/qwen2.5-coder-32b-instruct": "Qwen 2.5 Coder 32B (Agentic)",
    "nvidia/llama-3.1-nemotron-70b-instruct": "Nemotron 70B (Agentic)",
    "meta/llama-3.3-70b-instruct": "Llama 3.3 70B (Agentic)",
}

# All models (tool-capable get agentic mode, others get plain chat)
ALL_MODELS = {
    **TOOL_CAPABLE_MODELS,
    "deepseek-ai/deepseek-r1": "DeepSeek R1 (Chat only)",
    "mistralai/mistral-large-2-instruct": "Mistral Large 2 (Chat only)",
}

RECOMMENDED_MODEL = "nvidia/llama-3.1-nemotron-70b-instruct"

# Ensure workspace exists
Path(WORKSPACE_DIR).mkdir(parents=True, exist_ok=True)

# ---------------------------------------------------------------------------
# NIM Client
# ---------------------------------------------------------------------------

nim_client = AsyncOpenAI(
    base_url="https://integrate.api.nvidia.com/v1",
    api_key=NIM_API_KEY,
)

# ---------------------------------------------------------------------------
# Rate Limiting & Multi-Provider Setup
# ---------------------------------------------------------------------------

MISTRAL_API_KEY = os.environ.get("MISTRAL_API_KEY", "YyZD5l1SCwL83hJIKsRVF5I7Oklcxu4k")
mistral_client = AsyncOpenAI(
    base_url="https://api.mistral.ai/v1",
    api_key=MISTRAL_API_KEY if MISTRAL_API_KEY else "dummy_key",
) if MISTRAL_API_KEY else None

class MultiProviderRateLimiter:
    def __init__(self):
        self.nim_limit = 40
        self.nim_window = 60
        self.nim_calls = []
        self.mistral_last_call = 0.0
        self.lock = asyncio.Lock()

    async def wait_for_mistral(self):
        async with self.lock:
            now = time.time()
            elapsed = now - self.mistral_last_call
            if elapsed < 1.0:
                await asyncio.sleep(1.0 - elapsed)
            self.mistral_last_call = time.time()

    async def wait_for_nim(self):
        async with self.lock:
            now = time.time()
            self.nim_calls = [t for t in self.nim_calls if now - t < self.nim_window]
            if len(self.nim_calls) >= self.nim_limit - 2:
                sleep_time = self.nim_window - (now - self.nim_calls[0])
                print(f"[RateLimiter] Approaching NIM rate limit (40 RPM). Sleeping {sleep_time:.2f}s...")
                await asyncio.sleep(sleep_time)
            self.nim_calls.append(time.time())

rate_limiter = MultiProviderRateLimiter()

# ---------------------------------------------------------------------------
# Tool Definitions (OpenAI function calling format)
# ---------------------------------------------------------------------------

TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "read_file",
            "description": "Read the contents of a file. Use this to inspect existing code, configs, or any text file.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Relative path to the file from the workspace root"
                    }
                },
                "required": ["path"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "write_file",
            "description": "Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Creates parent directories automatically.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Relative path to the file from the workspace root"
                    },
                    "content": {
                        "type": "string",
                        "description": "The full content to write to the file"
                    }
                },
                "required": ["path", "content"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "run_bash",
            "description": "Execute a bash command in the workspace directory. Use for installing packages, running scripts, git operations, etc. Commands run with a 30 second timeout.",
            "parameters": {
                "type": "object",
                "properties": {
                    "command": {
                        "type": "string",
                        "description": "The bash command to execute"
                    }
                },
                "required": ["command"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "list_directory",
            "description": "List files and directories in a given path. Shows file sizes and directory markers.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Relative path to the directory from workspace root. Use '.' for the workspace root."
                    }
                },
                "required": ["path"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "grep_search",
            "description": "Search for a pattern in files within the workspace. Returns matching lines with file paths and line numbers.",
            "parameters": {
                "type": "object",
                "properties": {
                    "pattern": {
                        "type": "string",
                        "description": "The search pattern (supports basic regex)"
                    },
                    "path": {
                        "type": "string",
                        "description": "Directory or file to search in, relative to workspace root. Defaults to '.'",
                    }
                },
                "required": ["pattern"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "web_search",
            "description": "Search the web for up-to-date information, news, papers, or documentation.",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {
                        "type": "string",
                        "description": "The search query (be specific and detailed)"
                    }
                },
                "required": ["query"]
            }
        }
    },
    {
        "type": "function",
        "function": {
            "name": "web_read",
            "description": "Read the clean markdown content of any webpage URL to get detailed context, articles, documentation, or code.",
            "parameters": {
                "type": "object",
                "properties": {
                    "url": {
                        "type": "string",
                        "description": "The absolute URL of the webpage to read"
                    }
                },
                "required": ["url"]
            }
        }
    },
]

# ---------------------------------------------------------------------------
# Tool Execution
# ---------------------------------------------------------------------------

def _safe_path(rel_path: str) -> Path:
    """Resolve a relative path safely within the workspace."""
    workspace = Path(workspace_var.get()).resolve()
    target = (workspace / rel_path).resolve()
    # Prevent path traversal
    if not str(target).startswith(str(workspace)):
        raise ValueError(f"Path traversal detected: {rel_path}")
    return target


def repair_arguments(func_name: str, args: dict) -> tuple[dict, list[str]]:
    notes = []
    repaired_args = dict(args)
    
    # 1. Nesting extraction (e.g. {"path": {"path": "file.txt"}})
    for key in list(repaired_args.keys()):
        val = repaired_args[key]
        if isinstance(val, dict) and key in val:
            repaired_args[key] = val[key]
            notes.append(f"Flattened nested parameter '{key}'")
            
    # 2. Markdown stripping from bash command
    if func_name == "run_bash" and "command" in repaired_args:
        cmd = repaired_args["command"]
        if isinstance(cmd, str):
            pattern = r"```(?:bash)?\s*(.*?)\s*```"
            match = re.search(pattern, cmd, re.DOTALL)
            if match:
                repaired_args["command"] = match.group(1).strip()
                notes.append("Stripped markdown code blocks from bash command")
                
    # 3. Stringified array conversion
    for key, val in repaired_args.items():
        if isinstance(val, str) and val.strip().startswith("[") and val.strip().endswith("]"):
            try:
                parsed_arr = json.loads(val)
                if isinstance(parsed_arr, list):
                    repaired_args[key] = parsed_arr
                    notes.append(f"Converted stringified array for parameter '{key}' to native array")
            except:
                pass
                
    # 4. Optional empty objects replacing Null
    for key in list(repaired_args.keys()):
        if repaired_args[key] == {}:
            repaired_args[key] = None
            notes.append(f"Replaced empty object for parameter '{key}' with null")
            
    return repaired_args, notes


# ---------------------------------------------------------------------------
# Search and MCP Helpers
# ---------------------------------------------------------------------------

def sanitize_function_name(name: str) -> str:
    sanitized = re.sub(r'[^a-zA-Z0-9_\.\-]', '_', name)
    if not re.match(r'^[a-zA-Z_]', sanitized):
        sanitized = '_' + sanitized
    if len(sanitized) > 124:
        sanitized = sanitized[:124]
    return sanitized

def create_mcp_tool_id(server_name: str, tool_name: str) -> str:
    san_server = sanitize_function_name(server_name)
    san_tool = sanitize_function_name(tool_name)
    max_len = 124
    sep = "_"
    if len(san_server) + len(san_tool) + len(sep) > max_len:
        total = len(san_server) + len(san_tool)
        server_portion = int((len(san_server) / total) * (max_len - len(sep)))
        tool_portion = max_len - len(sep) - server_portion
        return f"{san_server[:server_portion]}{sep}{san_tool[:tool_portion]}"
    return f"{san_server}{sep}{san_tool}"

async def get_active_mcp_tools():
    """
    Queries `mcp_server` table and returns:
      1. List of OpenAI function definitions to append to dynamic tools list.
      2. Dictionary mapping `mcp_tool_id` to `(server_name, original_tool_name, config)`.
    """
    mcp_tools_list = []
    mcp_mapping = {}
    if not db_pool:
        return mcp_tools_list, mcp_mapping
    try:
        async with db_pool.acquire() as conn:
            rows = await conn.fetch("SELECT name, config, tool_info FROM mcp_server WHERE enabled = true")
            for row in rows:
                server_name = row["name"]
                config_raw = row["config"]
                if isinstance(config_raw, str):
                    config = json.loads(config_raw)
                else:
                    config = config_raw
                tool_info_raw = row["tool_info"]
                if not tool_info_raw:
                    continue
                if isinstance(tool_info_raw, str):
                    tool_info = json.loads(tool_info_raw)
                else:
                    tool_info = tool_info_raw
                for tool in tool_info:
                    tool_name = tool.get("name")
                    description = tool.get("description", "")
                    input_schema = tool.get("inputSchema", {})
                    tool_id = create_mcp_tool_id(server_name, tool_name)
                    mcp_mapping[tool_id] = (server_name, tool_name, config)
                    mcp_tools_list.append({
                        "type": "function",
                        "function": {
                            "name": tool_id,
                            "description": f"[from MCP server: {server_name}] {description}",
                            "parameters": input_schema
                        }
                    })
    except Exception as e:
        log_activity(f"[MCP Database Query Warning] Failed to load MCP tools: {e}")
    return mcp_tools_list, mcp_mapping

async def execute_web_search(query: str) -> str:
    tavily_key = os.environ.get("TAVILY_API_KEY", "")
    if tavily_key:
        try:
            async with httpx.AsyncClient(timeout=15.0) as client:
                r = await client.post("https://api.tavily.com/search", json={
                    "api_key": tavily_key,
                    "query": query,
                    "search_depth": "basic",
                    "max_results": 5
                })
                if r.status_code == 200:
                    data = r.json()
                    results = []
                    for item in data.get("results", []):
                        results.append(f"Title: {item.get('title')}\nURL: {item.get('url')}\nContent: {item.get('content')}\n---")
                    return "\n".join(results) if results else "No results found."
        except Exception as e:
            log_activity(f"[Tavily Error] {e}")

    exa_key = os.environ.get("EXA_API_KEY", "")
    if exa_key:
        try:
            async with httpx.AsyncClient(timeout=15.0) as client:
                r = await client.post("https://api.exa.ai/search", headers={
                    "x-api-key": exa_key,
                    "Content-Type": "application/json"
                }, json={
                    "query": query,
                    "numResults": 5,
                    "text": True
                })
                if r.status_code == 200:
                    data = r.json()
                    results = []
                    for item in data.get("results", []):
                        results.append(f"Title: {item.get('title')}\nURL: {item.get('url')}\nContent: {item.get('text', '')[:2000]}\n---")
                    return "\n".join(results) if results else "No results found."
        except Exception as e:
            log_activity(f"[Exa Error] {e}")

    try:
        escaped_query = urllib.parse.quote(query)
        async with httpx.AsyncClient(headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}, timeout=10.0) as client:
            r = await client.get(f"https://html.duckduckgo.com/html/?q={escaped_query}")
            if r.status_code == 200:
                try:
                    from bs4 import BeautifulSoup
                    soup = BeautifulSoup(r.text, 'html.parser')
                    results = []
                    for a in soup.find_all('a', class_='result__snippet')[:5]:
                        title_el = a.find_previous('a', class_='result__url')
                        title = title_el.text.strip() if title_el else "No Title"
                        url = title_el['href'] if title_el and 'href' in title_el.attrs else ""
                        snippet = a.text.strip()
                        results.append(f"Title: {title}\nURL: {url}\nSnippet: {snippet}\n---")
                    return "\n".join(results) if results else "No results found."
                except Exception:
                    snippets = re.findall(r'<a class="result__snippet"[^>]*>(.*?)</a>', r.text, re.DOTALL)
                    results = []
                    for s in snippets[:5]:
                        clean_s = re.sub(r'<[^>]*>', '', s).strip()
                        results.append(f"Snippet: {clean_s}\n---")
                    return "\n".join(results) if results else "No results found."
    except Exception as e:
        log_activity(f"[DDG Scrape Error] {e}")
    return "Error: Web search failed. Setup API keys (TAVILY_API_KEY, EXA_API_KEY) for best results."

async def execute_web_read(url: str) -> str:
    jina_url = f"https://r.jina.ai/{url}"
    try:
        async with httpx.AsyncClient(timeout=20.0) as client:
            r = await client.get(jina_url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
            if r.status_code == 200:
                content = r.text
                if len(content) > 30000:
                    content = content[:30000] + "\n\n[Truncated - webpage content is extremely long]"
                return content
    except Exception as e:
        log_activity(f"[Jina Reader Error] {e}")

    try:
        async with httpx.AsyncClient(timeout=15.0) as client:
            r = await client.get(url, headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"})
            if r.status_code == 200:
                html = r.text
                clean = re.sub(r'<script.*?>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
                clean = re.sub(r'<style.*?>.*?</style>', '', clean, flags=re.DOTALL | re.IGNORECASE)
                clean = re.sub(r'<.*?>', ' ', clean)
                clean = re.sub(r'\s+', ' ', clean).strip()
                if len(clean) > 15000:
                    clean = clean[:15000] + "\n\n[Truncated]"
                return clean
    except Exception as e:
        log_activity(f"[Direct Scrape Error] {e}")
    return f"Error: Failed to read URL {url}."


async def execute_tool(name: str, arguments: dict, mcp_mapping: dict = None) -> str:
    """Execute a tool and return its output as a string asynchronously."""
    try:
        if name == "read_file":
            path = _safe_path(arguments["path"])
            if not path.exists():
                return f"Error: File not found: {arguments['path']}"
            if not path.is_file():
                return f"Error: Not a file: {arguments['path']}"
            content = path.read_text(encoding="utf-8", errors="replace")
            if len(content) > 50000:
                return content[:50000] + f"\n\n[Truncated — file is {len(content)} chars]"
            return content

        elif name == "write_file":
            path = _safe_path(arguments["path"])
            path.parent.mkdir(parents=True, exist_ok=True)
            path.write_text(arguments["content"], encoding="utf-8")
            return f"Successfully wrote {len(arguments['content'])} chars to {arguments['path']}"

        elif name == "run_bash":
            command = arguments["command"]
            # Safety: block dangerous commands
            blocked = ["rm -rf /", "mkfs", "dd if=", ":(){", "fork bomb"]
            if any(b in command.lower() for b in blocked):
                return "Error: Command blocked for safety reasons"
            
            # ASYNC SUBPROCESS - This prevents the FastAPI server from freezing!
            process = await asyncio.create_subprocess_shell(
                command,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
                cwd=workspace_var.get(),
                env={**os.environ, "HOME": "/tmp", "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin")},
            )
            
            try:
                stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=30)
                output = ""
                if stdout:
                    output += stdout.decode('utf-8', errors='replace')
                if stderr:
                    output += ("\n" if output else "") + f"[stderr] {stderr.decode('utf-8', errors='replace')}"
                if process.returncode != 0:
                    output += f"\n[exit code: {process.returncode}]"
                if not output:
                    output = "[command completed with no output]"
            except asyncio.TimeoutError:
                try:
                    process.kill()
                except Exception:
                    pass
                await process.communicate()
                return "Error: Command timed out after 30 seconds"
                
            if len(output) > 20000:
                output = output[:20000] + f"\n\n[Truncated — output is {len(output)} chars]"
            return output

        elif name == "list_directory":
            path = _safe_path(arguments.get("path", "."))
            if not path.exists():
                return f"Error: Directory not found: {arguments.get('path', '.')}"
            if not path.is_dir():
                return f"Error: Not a directory: {arguments.get('path', '.')}"
            entries = []
            for item in sorted(path.iterdir()):
                if item.is_dir():
                    entries.append(f"  📁 {item.name}/")
                else:
                    size = item.stat().st_size
                    if size < 1024:
                        size_str = f"{size}B"
                    elif size < 1024 * 1024:
                        size_str = f"{size/1024:.1f}KB"
                    else:
                        size_str = f"{size/(1024*1024):.1f}MB"
                    entries.append(f"  📄 {item.name} ({size_str})")
            return f"Contents of {arguments.get('path', '.')}:\n" + "\n".join(entries) if entries else "Empty directory"

        elif name == "grep_search":
            pattern = arguments["pattern"]
            search_path = arguments.get("path", ".")
            path = _safe_path(search_path)
            
            process = await asyncio.create_subprocess_exec(
                "grep", "-rn", "--include=*", pattern, str(path),
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
                cwd=workspace_var.get(),
            )
            try:
                stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=10)
                output = stdout.decode('utf-8', errors='replace') if stdout else "No matches found"
            except asyncio.TimeoutError:
                try:
                    process.kill()
                except Exception:
                    pass
                await process.communicate()
                return "Error: Grep search timed out"
                
            if len(output) > 10000:
                output = output[:10000] + "\n\n[Truncated]"
            return output

        elif name == "web_search":
            query = arguments.get("query")
            return await execute_web_search(query)

        elif name == "web_read":
            url = arguments.get("url")
            return await execute_web_read(url)

        elif mcp_mapping and name in mcp_mapping:
            server_name, original_tool_name, config = mcp_mapping[name]
            try:
                # Ensure registered on Space 4
                await apost_json(f"{SPACE4_URL}/api/mcp/register", {
                    "name": server_name,
                    "config": config
                })
                # Call tool on Space 4
                call_res = await apost_json(f"{SPACE4_URL}/api/mcp/call", {
                    "serverName": server_name,
                    "toolName": original_tool_name,
                    "arguments": arguments
                })
                if isinstance(call_res, dict) and "error" in call_res:
                    return f"Error from MCP server {server_name}: {call_res['error']}"
                if isinstance(call_res, dict) and "content" in call_res:
                    texts = []
                    for item in call_res["content"]:
                        if isinstance(item, dict) and item.get("type") == "text":
                            texts.append(item.get("text", ""))
                    return "\n".join(texts)
                return str(call_res)
            except Exception as e:
                return f"Error executing MCP tool {name}: {str(e)}"

        else:
            return f"Error: Unknown tool: {name}"

    except Exception as e:
        return f"Error executing {name}: {str(e)}"


# ---------------------------------------------------------------------------
# Database (Session Persistence)
# ---------------------------------------------------------------------------

db_pool: Optional[asyncpg.Pool] = None


async def init_db():
    """Initialize database connection pool and create tables."""
    global db_pool
    if not DATABASE_URL:
        return
    try:
        db_pool = await asyncpg.create_pool(
            DATABASE_URL, 
            ssl="require", 
            min_size=1, 
            max_size=3,
            max_inactive_connection_lifetime=300
        )
        async with db_pool.acquire() as conn:
            await conn.execute("""
                CREATE TABLE IF NOT EXISTS agent_session_entries (
                    id          BIGSERIAL PRIMARY KEY,
                    project_key TEXT        NOT NULL,
                    session_id  TEXT        NOT NULL,
                    subpath     TEXT,
                    entry       JSONB       NOT NULL,
                    created_at  TIMESTAMPTZ NOT NULL DEFAULT now()
                );
                CREATE INDEX IF NOT EXISTS idx_session_key ON agent_session_entries (project_key, session_id, subpath, id);
                CREATE INDEX IF NOT EXISTS idx_project_session ON agent_session_entries (project_key, session_id);
                
                CREATE TABLE IF NOT EXISTS eternity_projects (
                    id           SERIAL PRIMARY KEY,
                    project_name VARCHAR(100) UNIQUE NOT NULL,
                    goal         TEXT NOT NULL,
                    deadline     TIMESTAMPTZ NOT NULL,
                    current_mode VARCHAR(20) NOT NULL DEFAULT 'build',
                    is_active    BOOLEAN NOT NULL DEFAULT true,
                    priority     VARCHAR(20) NOT NULL DEFAULT 'low',
                    roadmap      JSONB DEFAULT '[]',
                    latest_brief TEXT,
                    created_at   TIMESTAMPTZ NOT NULL DEFAULT now()
                );
            """)
    except Exception as e:
        print(f"[DB] Warning: Could not initialize database: {e}")
        db_pool = None


async def save_message(session_id: str, role: str, content: str = None, 
                       tool_calls: list = None, tool_call_id: str = None):
    """Save a message to the session store using the unified schema."""
    if not db_pool:
        return
    msg = {"role": role}
    if content is not None:
        msg["content"] = content
    if tool_calls:
        msg["tool_calls"] = tool_calls
    if tool_call_id:
        msg["tool_call_id"] = tool_call_id
        
    try:
        async with db_pool.acquire() as conn:
            await conn.execute(
                "INSERT INTO agent_session_entries (project_key, session_id, subpath, entry) VALUES ($1, $2, $3, $4)",
                "fastapi-completions",
                session_id,
                None,
                json.dumps(msg)
            )
    except Exception as e:
        print(f"[DB] Warning: Could not save message: {e}")


async def load_session(session_id: str) -> list:
    """Load conversation history from the session store using the unified schema."""
    if not db_pool:
        return []
    try:
        async with db_pool.acquire() as conn:
            rows = await conn.fetch(
                "SELECT entry FROM agent_session_entries WHERE project_key = $1 AND session_id = $2 AND subpath IS NOT DISTINCT FROM $3 ORDER BY id",
                "fastapi-completions",
                session_id,
                None
            )
            return [json.loads(row["entry"]) for row in rows]
    except Exception as e:
        print(f"[DB] Warning: Could not load session: {e}")
        return []


# ---------------------------------------------------------------------------
# SSE Chunk Formatting (OpenAI delta format)
# ---------------------------------------------------------------------------

def make_chunk(request_id: str, model: str, content: str = "", finish_reason: str = None) -> str:
    """Create an OpenAI-compatible SSE chunk."""
    delta = {}
    if content:
        delta["content"] = content
    if finish_reason and not content:
        delta = {}
    
    chunk = {
        "id": f"chatcmpl-{request_id}",
        "object": "chat.completion.chunk",
        "created": int(time.time()),
        "model": model,
        "choices": [{
            "index": 0,
            "delta": delta,
            "finish_reason": finish_reason,
        }],
    }
    return f"data: {json.dumps(chunk)}\n\n"


# ---------------------------------------------------------------------------
# FastAPI Application
# ---------------------------------------------------------------------------

app = FastAPI(title="Claude Code Backend", version="1.0.0")
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_methods=["*"],
    allow_headers=["*"],
)


def auth(authorization: str = None):
    """Verify bearer token."""
    if not BACKEND_API_KEY:
        return  # No auth configured
    expected = f"Bearer {BACKEND_API_KEY}"
    if authorization != expected:
        raise HTTPException(status_code=401, detail="Unauthorized")


async def check_models_health():
    # Mark all models as statically ONLINE to save API RPM quotas
    for model in ALL_MODELS.keys():
        MODEL_STATUSES[model] = {"status": "ONLINE (Stable)", "latency": "Fast", "raw_latency": 0.1}
    log_activity("[Health Check] Zero-cost status check complete: all models marked ONLINE (No API calls made).")

@app.on_event("startup")
async def startup():
    await init_db()
    Path(WORKSPACE_DIR).mkdir(parents=True, exist_ok=True)
    # Initialize statuses for all models as ONLINE by default
    for model_id, display_name in ALL_MODELS.items():
        MODEL_STATUSES[model_id] = {"status": "ONLINE", "latency": "0.10s", "raw_latency": 0.10}
    log_activity(f"FastAPI backend started. Workspace: {WORKSPACE_DIR}")


# ---------------------------------------------------------------------------
# /v1/chat/completions — Main endpoint
# ---------------------------------------------------------------------------

AGENTIC_SYSTEM_PROMPT = """You are an expert coding assistant with access to tools for file operations and command execution. 

When the user asks you to create, edit, or debug code:
1. Use `list_directory` and `read_file` to understand the current state
2. Use `write_file` to create or modify files
3. Use `run_bash` to execute commands (install packages, run scripts, test code)
4. Use `grep_search` to find patterns in code

IMPORTANT RULES:
- Always use tools to take action. Do NOT just describe what to do — actually DO it.
- After writing code, run it to verify it works.
- If a command fails, read the error and fix it.
- Work in the /tmp/workspace directory.
- Be concise in your explanations, but thorough in your tool usage.
- You are running in a multi-round tool-execution loop. Do NOT output redundant greetings (e.g. "Hey saumil!", "Let me check...", "Ready to build...") in intermediate rounds. Only output them in your first response if appropriate. Be direct and proceed with tool calls.
"""


def compact_history(messages: list) -> list:
    """
    Bulletproof conversation history compaction.
    Guarantees that the total context length stays well below the model's token limits.
    """
    if not messages:
        return []

    # Find and preserve the system prompt
    system_msg = None
    if messages[0].get("role") == "system":
        system_msg = messages[0]
        start_idx = 1
    else:
        start_idx = 0

    other_messages = messages[start_idx:]
    
    # 1. Truncate individually massive messages (e.g. file contents or massive bash outputs)
    # Even recent messages should be compacted if they are ridiculously large!
    compacted_others = []
    for msg in other_messages:
        role = msg.get("role")
        content = msg.get("content") or ""
        msg_copy = dict(msg)
        
        if len(content) > 15000:
            msg_copy["content"] = content[:5000] + f"\n\n[... Truncated {len(content) - 10000} characters to prevent model context limits from overflowing ...]\n\n" + content[-5000:]
        
        compacted_others.append(msg_copy)

    # 2. Enforce total character size budget (max ~300,000 characters / ~75,000 tokens)
    # Iterate backwards from newest to oldest
    final_list = []
    total_chars = 0
    max_budget = 300000
    
    for msg in reversed(compacted_others):
        msg_len = len(msg.get("content") or "")
        if total_chars + msg_len < max_budget or len(final_list) < 2:
            final_list.append(msg)
            total_chars += msg_len
        else:
            # Drop older messages once we exceed context window budget
            pass

    # Reverse back to chronological order
    final_list.reverse()
    
    if system_msg:
        final_list.insert(0, system_msg)
        
    if len(messages) != len(final_list):
        log_activity(f"[Auto-Compaction] Sliced context window from {len(messages)} down to {len(final_list)} messages (Total chars: {total_chars})")
        
    return final_list


@app.post("/v1/chat/completions")
async def chat_completions(request: Request, authorization: str = Header(None)):
    auth(authorization)
    body = await request.json()

    requested_model = body.get("model", "meta/llama-3.1-70b-instruct")
    messages = body.get("messages", [])
    stream = body.get("stream", False)
    session_id = body.get("session_id") or str(uuid.uuid4())
    
    # Route Mistral queries natively to the Mistral API
    client = nim_client
    if "mistral" in requested_model.lower():
        if mistral_client:
            client = mistral_client
            if requested_model == "mistralai/mistral-large-2-instruct":
                requested_model = "mistral-large-latest"
    
    is_agentic = requested_model in TOOL_CAPABLE_MODELS
    request_id = str(uuid.uuid4())[:8]

    ACTIVE_SESSIONS.add(session_id)
    log_activity(f"Session [{session_id[:6]}] connected. Model: {requested_model} | Provider: {'Mistral' if client == mistral_client else 'NIM'}")

    # Build message history
    final_messages = []
    
    # Add agentic system prompt for tool-capable models
    if is_agentic:
        # Check if there's already a system message
        has_system = any(m.get("role") == "system" for m in messages)
        if has_system:
            # Prepend agentic prompt to existing system message
            for m in messages:
                if m["role"] == "system":
                    final_messages.append({
                        "role": "system",
                        "content": AGENTIC_SYSTEM_PROMPT + "\n\nAdditional instructions:\n" + m["content"]
                    })
                else:
                    final_messages.append(m)
        else:
            final_messages.append({"role": "system", "content": AGENTIC_SYSTEM_PROMPT})
            final_messages.extend(messages)
    else:
        final_messages = list(messages)

    # Perform auto-compaction before executing agent loops
    final_messages = compact_history(final_messages)

    # Save the user's message to DB
    user_msg = next((m for m in reversed(messages) if m.get("role") == "user"), None)
    if user_msg:
        await save_message(session_id, "user", user_msg.get("content", ""))

    session_folder = session_id[:8] if session_id else "default"
    active_workspace = os.path.join(WORKSPACE_DIR, session_folder)
    os.makedirs(active_workspace, exist_ok=True)

    if not stream:
        # Non-streaming: simple completion
        workspace_token = workspace_var.set(active_workspace)
        try:
            mcp_tools, mcp_mapping = await get_active_mcp_tools()
            active_tools = list(TOOLS) + mcp_tools
            kwargs = {"model": requested_model, "messages": final_messages}
            if is_agentic:
                kwargs["tools"] = active_tools
                kwargs["tool_choice"] = "auto"
            async with completions_semaphore:
                response = await client.chat.completions.create(**kwargs)
            content = response.choices[0].message.content or ""
            await save_message(session_id, "assistant", content)
            ACTIVE_SESSIONS.discard(session_id)
            log_activity(f"Session [{session_id[:6]}] finished (non-streaming)")
            return JSONResponse({
                "id": f"chatcmpl-{request_id}",
                "object": "chat.completion",
                "created": int(time.time()),
                "model": requested_model,
                "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
            })
        except Exception as e:
            ACTIVE_SESSIONS.discard(session_id)
            return JSONResponse({"error": {"message": str(e), "type": "internal_error"}}, status_code=500)
        finally:
            workspace_var.reset(workspace_token)

    # Streaming + agentic loop
    async def generate() -> AsyncIterator[str]:
        nonlocal final_messages
        workspace_token = workspace_var.set(active_workspace)
        try:
            mcp_tools, mcp_mapping = await get_active_mcp_tools()
            active_tools = list(TOOLS) + mcp_tools
            async with completions_semaphore:
                for round_num in range(MAX_TOOL_ROUNDS + 1):
                    # Perform auto-compaction before calling NIM API
                    final_messages = compact_history(final_messages)
                    kwargs = {"model": requested_model, "messages": final_messages, "stream": True}
                    if is_agentic:
                        kwargs["tools"] = active_tools
                        kwargs["tool_choice"] = "auto"

                    # Collect streamed response
                    full_content = ""
                    tool_calls_raw = {}  # index -> {id, name, arguments_str}
                    
                    async for chunk in await client.chat.completions.create(**kwargs):
                        choice = chunk.choices[0] if chunk.choices else None
                        if not choice:
                            continue
                        delta = choice.delta
                        
                        # Stream text content to client
                        if delta and delta.content:
                            full_content += delta.content
                            yield make_chunk(request_id, requested_model, delta.content)
                        
                        # Collect tool calls
                        if delta and delta.tool_calls:
                            for tc in delta.tool_calls:
                                idx = tc.index
                                if idx not in tool_calls_raw:
                                    tool_calls_raw[idx] = {
                                        "id": tc.id or f"call_{uuid.uuid4().hex[:8]}",
                                        "name": tc.function.name if tc.function and tc.function.name else "",
                                        "arguments": ""
                                    }
                                if tc.function and tc.function.name:
                                    tool_calls_raw[idx]["name"] = tc.function.name
                                if tc.id:
                                    tool_calls_raw[idx]["id"] = tc.id
                                if tc.function and tc.function.arguments:
                                    tool_calls_raw[idx]["arguments"] += tc.function.arguments
                        
                        # Check for finish
                        if choice.finish_reason == "stop":
                            break
                        if choice.finish_reason == "tool_calls":
                            break

                    # If no tool calls, we're done
                    if not tool_calls_raw:
                        await save_message(session_id, "assistant", full_content)
                        yield make_chunk(request_id, requested_model, finish_reason="stop")
                        yield "data: [DONE]\n\n"
                        return

                    # Execute tool calls
                    tool_calls_list = []
                    for idx in sorted(tool_calls_raw.keys()):
                        tc = tool_calls_raw[idx]
                        tool_calls_list.append({
                            "id": tc["id"],
                            "type": "function",
                            "function": {"name": tc["name"], "arguments": tc["arguments"]}
                        })

                    # Add assistant message with tool calls to history
                    assistant_msg = {"role": "assistant", "content": full_content or None, "tool_calls": tool_calls_list}
                    final_messages.append(assistant_msg)

                    # Execute each tool and add results
                    for tc in tool_calls_list:
                        func_name = tc["function"]["name"]
                        raw_args_str = tc["function"]["arguments"]
                        try:
                            func_args = json.loads(raw_args_str)
                        except json.JSONDecodeError:
                            # Attempt raw JSON repair
                            repaired_str = raw_args_str.strip()
                            if not repaired_str.startswith("{"):
                                repaired_str = "{" + repaired_str
                            if not repaired_str.endswith("}"):
                                repaired_str = repaired_str + "}"
                            try:
                                func_args = json.loads(repaired_str)
                                log_activity(f"Auto-fixed invalid JSON string for tool: {func_name}")
                            except:
                                func_args = {}

                        # Perform semantic repairs
                        repaired_args, repair_notes = repair_arguments(func_name, func_args)
                    
                        # Log activity
                        log_activity(f"Tool execution: {func_name} args={repaired_args}")
                        if repair_notes:
                            for note in repair_notes:
                                log_activity(f"[Tool Repair] {note}")

                        # Show tool execution to user
                        yield make_chunk(request_id, requested_model, f"\n\n🔧 **{func_name}**")
                        if repair_notes:
                            yield make_chunk(request_id, requested_model, " *(Auto-Repaired)*")
                        if func_name == "run_bash" and "command" in repaired_args:
                            yield make_chunk(request_id, requested_model, f": `{repaired_args['command']}`\n")
                        elif func_name == "read_file" and "path" in repaired_args:
                            yield make_chunk(request_id, requested_model, f": `{repaired_args['path']}`\n")
                        elif func_name == "write_file" and "path" in repaired_args:
                            yield make_chunk(request_id, requested_model, f": `{repaired_args['path']}`\n")
                        elif func_name == "list_directory":
                            yield make_chunk(request_id, requested_model, f": `{repaired_args.get('path', '.')}`\n")
                        elif func_name == "grep_search":
                            yield make_chunk(request_id, requested_model, f": `{repaired_args.get('pattern', '')}`\n")
                        elif func_name == "web_search" and "query" in repaired_args:
                            yield make_chunk(request_id, requested_model, f": `{repaired_args['query']}`\n")
                        elif func_name == "web_read" and "url" in repaired_args:
                            yield make_chunk(request_id, requested_model, f": `{repaired_args['url']}`\n")
                        elif mcp_mapping and func_name in mcp_mapping:
                            yield make_chunk(request_id, requested_model, f": calling tool\n")
                        else:
                            yield make_chunk(request_id, requested_model, "\n")

                        # Execute the tool
                        result = await execute_tool(func_name, repaired_args, mcp_mapping)

                        # Append teaching note if repaired
                        if repair_notes:
                            result += f"\n\n[SYSTEM REPAIR NOTE: The harness automatically fixed formatting issues: {', '.join(repair_notes)}. Please strictly follow the tool's JSON schema in subsequent calls without these wrapping/formatting errors.]"

                        # Show truncated result to user
                        preview = result[:500] + ("..." if len(result) > 500 else "")
                        yield make_chunk(request_id, requested_model, f"```\n{preview}\n```\n")

                        # Add tool result to message history
                        final_messages.append({
                            "role": "tool",
                            "tool_call_id": tc["id"],
                            "content": result,
                        })

                        await save_message(session_id, "tool", result, tool_call_id=tc["id"])

                # Continue the agentic loop (model processes tool results)

                # If we hit max rounds, finish
                yield make_chunk(request_id, requested_model, "\n\n⚠️ Reached maximum tool call rounds.")
                yield make_chunk(request_id, requested_model, finish_reason="stop")
                yield "data: [DONE]\n\n"

        except Exception as e:
            error_msg = f"\n\n❌ Error: {str(e)}"
            yield make_chunk(request_id, requested_model, error_msg)
            yield make_chunk(request_id, requested_model, finish_reason="stop")
            yield "data: [DONE]\n\n"
        finally:
            workspace_var.reset(workspace_token)

    return StreamingResponse(
        generate(),
        media_type="text/event-stream",
        headers={
            "Cache-Control": "no-cache",
            "X-Accel-Buffering": "no",
            "Connection": "keep-alive",
        },
    )


# ---------------------------------------------------------------------------
# /v1/models — Model listing
# ---------------------------------------------------------------------------

@app.get("/v1/models")
async def list_models(authorization: str = Header(None)):
    auth(authorization)
    models = []
    for model_id, display_name in ALL_MODELS.items():
        models.append({
            "id": model_id,
            "object": "model",
            "created": 1700000000,
            "owned_by": "nvidia-nim",
            "permission": [],
            "root": model_id,
            "parent": None,
        })
    return {"object": "list", "data": models}


# ---------------------------------------------------------------------------
# /health — Health check
# ---------------------------------------------------------------------------

@app.get("/api/workspace/tree")
async def get_workspace_tree():
    def build_tree(current_path: Path, relative_to: Path) -> dict:
        name = current_path.name
        try:
            rel_path = str(current_path.relative_to(relative_to)).replace("\\", "/")
        except ValueError:
            rel_path = ""
        if rel_path == ".":
            rel_path = ""
            
        if current_path.is_dir():
            children = []
            try:
                for child in sorted(current_path.iterdir(), key=lambda x: (not x.is_dir(), x.name)):
                    if child.name in [".git", "node_modules", ".next", "__pycache__", ".agents", ".gemini"]:
                        continue
                    children.append(build_tree(child, relative_to))
            except Exception:
                pass
            return {
                "name": name or "workspace",
                "path": rel_path,
                "type": "directory",
                "children": children
            }
        else:
            return {
                "name": name,
                "path": rel_path,
                "type": "file",
                "size": current_path.stat().st_size if current_path.exists() else 0
            }
            
    try:
        active_project = ""
        if db_pool:
            try:
                async with db_pool.acquire() as conn:
                    row = await conn.fetchrow("SELECT project_name FROM eternity_projects WHERE is_active = true ORDER BY id DESC LIMIT 1")
                    if row:
                        active_project = row["project_name"].replace(" ", "-").lower()
            except Exception:
                pass
        
        project_folder = active_project if active_project else "default"
        w_path = (Path(WORKSPACE_DIR) / project_folder).resolve()
        if not w_path.exists():
            w_path.mkdir(parents=True, exist_ok=True)
        return build_tree(w_path, w_path)
    except Exception as e:
        return {"error": str(e)}


@app.get("/api/workspace/file")
async def get_workspace_file(path: str):
    try:
        active_project = ""
        if db_pool:
            try:
                async with db_pool.acquire() as conn:
                    row = await conn.fetchrow("SELECT project_name FROM eternity_projects WHERE is_active = true ORDER BY id DESC LIMIT 1")
                    if row:
                        active_project = row["project_name"].replace(" ", "-").lower()
            except Exception:
                pass
        
        project_folder = active_project if active_project else "default"
        w_path = (Path(WORKSPACE_DIR) / project_folder).resolve()
        token = workspace_var.set(str(w_path))
        try:
            safe_p = _safe_path(path)
            if not safe_p.exists() or not safe_p.is_file():
                raise HTTPException(status_code=404, detail="File not found")
            content = safe_p.read_text(encoding="utf-8", errors="replace")
            return {"path": path, "content": content}
        finally:
            workspace_var.reset(token)
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


@app.get("/api/workspace/latest-screenshot")
async def get_latest_screenshot():
    try:
        active_project = ""
        if db_pool:
            try:
                async with db_pool.acquire() as conn:
                    row = await conn.fetchrow("SELECT project_name FROM eternity_projects WHERE is_active = true ORDER BY id DESC LIMIT 1")
                    if row:
                        active_project = row["project_name"].replace(" ", "-").lower()
            except Exception:
                pass
        
        project_folder = active_project if active_project else "default"
        w_path = (Path(WORKSPACE_DIR) / project_folder).resolve()
        png_files = []
        for p in w_path.rglob("*.png"):
            if any(part.startswith(".") for part in p.parts):
                continue
            try:
                mtime = p.stat().st_mtime
                png_files.append((p, mtime))
            except Exception:
                pass
        if not png_files:
            raise HTTPException(status_code=404, detail="No screenshot found")
        newest = max(png_files, key=lambda x: x[1])[0]
        from fastapi.responses import FileResponse
        return FileResponse(str(newest))
    except HTTPException as he:
        raise he
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


# ---------------------------------------------------------------------------
# Dashboard and Status API
# ---------------------------------------------------------------------------

DASHBOARD_HTML = """
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Claude Code Agent Console</title>
    <script src="https://cdn.tailwindcss.com"></script>
    <style>
        @import url('https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;700&family=Outfit:wght@400;600;800&display=swap');
        body {
            font-family: 'Outfit', sans-serif;
            background-color: #0b0c10;
        }
        .code-font {
            font-family: 'Fira Code', monospace;
        }
        .glow-amber {
            box-shadow: 0 0 15px rgba(245, 158, 11, 0.2);
        }
    </style>
</head>
<body class="text-gray-100 min-h-screen flex flex-col pb-10">
    <header class="border-b border-gray-800 bg-gray-950/80 backdrop-blur px-6 py-4 flex items-center justify-between sticky top-0 z-50">
        <div class="flex items-center space-x-3">
            <span class="text-2xl font-extrabold tracking-tight bg-gradient-to-r from-blue-400 via-indigo-400 to-purple-400 bg-clip-text text-transparent">
                Claude Code Agent Console
            </span>
            <span class="px-2 py-0.5 text-xs rounded bg-blue-500/10 text-blue-400 border border-blue-500/20 font-semibold animate-pulse">
                LIVE
            </span>
        </div>
        <div class="flex items-center space-x-4 text-sm text-gray-400">
            <div>Workspace: <span class="text-gray-200 code-font">/tmp/workspace</span></div>
            <div class="h-4 w-px bg-gray-800"></div>
            <div>Active Sessions: <span id="active-sessions-count" class="text-blue-400 font-bold code-font">0</span></div>
        </div>
    </header>

    <!-- Navigation Tabs -->
    <div class="border-b border-gray-800 max-w-7xl w-full mx-auto px-6 mt-6 flex space-x-6 text-sm">
        <button onclick="switchTab('models')" id="tab-btn-models" class="pb-3 border-b-2 border-blue-500 font-semibold text-blue-400 transition-all">NIM Models</button>
        <button onclick="switchTab('logs')" id="tab-btn-logs" class="pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold transition-all">Live Logs</button>
        <button onclick="switchTab('eternity')" id="tab-btn-eternity" class="pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold transition-all">Eternity R&D Lab</button>
        <button onclick="switchTab('explorer')" id="tab-btn-explorer" class="pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold flex items-center space-x-1 transition-all">
            <span>Workspace Explorer (IDE)</span>
            <span class="px-1.5 py-0.5 rounded bg-blue-500/10 text-blue-400 border border-blue-500/20 text-[10px] font-bold">VS Code View</span>
        </button>
    </div>

    <!-- MAIN SECTIONS -->
    <main class="max-w-7xl w-full mx-auto px-6 mt-8 flex-1">
        
        <!-- SECTION: Models -->
        <div id="section-models" class="space-y-6">
            <div class="flex items-center justify-between">
                <h2 class="text-lg font-bold tracking-tight text-gray-300">Nvidia NIM Models & Health Status</h2>
                <span class="text-xs text-gray-500">Checked every 15 mins</span>
            </div>
            <div id="models-container" class="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
                <!-- Dynamically loaded models go here -->
            </div>
        </div>

        <!-- SECTION: Logs -->
        <div id="section-logs" class="hidden space-y-6">
            <h2 class="text-lg font-bold tracking-tight text-gray-300">System Activity Logs</h2>
            <div class="border border-gray-800 rounded-lg overflow-hidden bg-gray-950 flex flex-col min-h-[500px]">
                <div class="bg-gray-900 px-4 py-2 border-b border-gray-800 flex items-center justify-between">
                    <span class="text-xs text-gray-400 font-semibold code-font">agent-stdout.log</span>
                    <div class="flex space-x-1.5">
                        <span class="w-2.5 h-2.5 rounded-full bg-red-500/30"></span>
                        <span class="w-2.5 h-2.5 rounded-full bg-yellow-500/30"></span>
                        <span class="w-2.5 h-2.5 rounded-full bg-green-500/30"></span>
                    </div>
                </div>
                <div id="terminal-content" class="p-4 flex-1 overflow-y-auto code-font text-xs text-green-400 bg-black/90 space-y-1 select-all h-[450px]">
                    <!-- Logs go here -->
                </div>
            </div>
        </div>

        <!-- SECTION: Workspace Explorer -->
        <div id="section-explorer" class="hidden space-y-6">
            <div class="flex items-center justify-between">
                <h2 class="text-lg font-bold tracking-tight text-gray-300">Visual Workspace IDE</h2>
                <button onclick="refreshFileTree()" class="text-xs px-2.5 py-1 rounded bg-blue-500/10 text-blue-400 border border-blue-500/20 hover:bg-blue-500/20 transition-all font-semibold">
                    🔄 Refresh Tree
                </button>
            </div>
            
            <div class="grid grid-cols-1 md:grid-cols-3 gap-6 border border-gray-800 rounded-xl bg-gray-950 overflow-hidden h-[600px]">
                <!-- File Tree Sidebar -->
                <div class="border-r border-gray-800 flex flex-col bg-gray-950 h-full">
                    <div class="px-4 py-2 border-b border-gray-800 bg-gray-900 text-xs font-semibold tracking-wider text-gray-400 code-font">
                        📁 EXPLORER: WORKSPACE
                    </div>
                    <div id="file-tree" class="p-3 flex-1 overflow-y-auto space-y-0.5 select-none">
                        <!-- Tree will be loaded here -->
                        <span class="text-xs text-gray-500 italic px-2">Loading directory tree...</span>
                    </div>
                </div>

                <!-- Editor panel -->
                <div class="md:col-span-2 flex flex-col bg-black/40 h-full">
                    <div class="px-4 py-2 border-b border-gray-800 bg-gray-900 flex items-center justify-between">
                        <span id="editor-title" class="text-xs font-semibold text-gray-400 code-font">📄 Welcome screen</span>
                        <div class="flex space-x-1.5">
                            <span class="w-2 h-2 rounded-full bg-gray-700"></span>
                            <span class="w-2 h-2 rounded-full bg-gray-700"></span>
                        </div>
                    </div>
                    <div class="flex-1 p-4 overflow-auto code-font text-xs text-gray-200">
                        <pre id="editor-content" class="whitespace-pre overflow-x-auto select-text h-[500px]">
Welcome to Claude Code Workspace Explorer.
Select a file from the sidebar explorer on the left to read its code contents in real-time.
                        </pre>
                    </div>
                </div>
            </div>
        </div>

        <!-- SECTION: Eternity Lab -->
        <div id="section-eternity" class="hidden space-y-6">
            <div class="flex items-center justify-between border-b border-gray-800 pb-4">
                <div>
                    <h2 class="text-xl font-bold tracking-tight text-gray-200">Eternity R&D Projects</h2>
                    <p class="text-xs text-gray-500 mt-1">Autonomous multi-agent loop orchestrator panel</p>
                </div>
                <button onclick="openNewProjectModal()" class="px-4 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded text-sm font-semibold transition-all shadow-lg glow-amber">
                    ➕ New R&D Goal
                </button>
            </div>
            
            <div id="eternity-projects-list" class="grid grid-cols-1 gap-6">
                <!-- Dynamically populated project cards -->
            </div>
        </div>

    </main>

    <!-- MODAL: New Project -->
    <div id="new-project-modal" class="fixed inset-0 bg-black/80 backdrop-blur-sm flex items-center justify-center hidden z-50">
        <div class="bg-gray-900 border border-gray-800 p-6 rounded-xl max-w-md w-full space-y-4">
            <h3 class="text-lg font-bold text-gray-100">Initialize R&D Goal</h3>
            <div class="space-y-3 text-sm">
                <div>
                    <label class="block text-gray-400 mb-1">Project Name (Letters/Numbers/Dashes only)</label>
                    <input id="proj-name" type="text" class="w-full bg-black border border-gray-800 rounded p-2 text-gray-200" placeholder="e.g. stoichiometry-solver">
                </div>
                <div>
                    <label class="block text-gray-400 mb-1">Problem Statement / Goal</label>
                    <textarea id="proj-goal" rows="4" class="w-full bg-black border border-gray-800 rounded p-2 text-gray-200" placeholder="Describe the goal in detail..."></textarea>
                </div>
                <div class="grid grid-cols-2 gap-4">
                    <div>
                        <label class="block text-gray-400 mb-1">Deadline (Hours)</label>
                        <input id="proj-deadline" type="number" step="0.01" value="1.0" class="w-full bg-black border border-gray-800 rounded p-2 text-gray-200">
                    </div>
                    <div>
                        <label class="block text-gray-400 mb-1">Priority</label>
                        <select id="proj-priority" class="w-full bg-black border border-gray-800 rounded p-2 text-gray-200">
                            <option value="supreme">Supreme Priority</option>
                            <option value="low" selected>Low Priority</option>
                        </select>
                    </div>
                </div>
            </div>
            <div class="flex justify-end space-x-3 text-sm pt-2">
                <button onclick="closeNewProjectModal()" class="px-4 py-2 border border-gray-800 text-gray-400 rounded hover:bg-gray-800">Cancel</button>
                <button onclick="submitNewProject()" class="px-4 py-2 bg-blue-600 hover:bg-blue-700 text-white rounded font-semibold">Start R&D</button>
            </div>
        </div>
    </div>

    <script>
        let currentTab = 'models';

        function switchTab(tabId) {
            currentTab = tabId;
            // Toggle sections
            document.getElementById('section-models').classList.add('hidden');
            document.getElementById('section-logs').classList.add('hidden');
            document.getElementById('section-explorer').classList.add('hidden');
            document.getElementById('section-eternity').classList.add('hidden');
            
            document.getElementById('tab-btn-models').className = 'pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold transition-all';
            document.getElementById('tab-btn-logs').className = 'pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold transition-all';
            document.getElementById('tab-btn-eternity').className = 'pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold transition-all';
            document.getElementById('tab-btn-explorer').className = 'pb-3 border-b-2 border-transparent text-gray-400 hover:text-gray-200 font-semibold flex items-center space-x-1 transition-all';

            if (tabId === 'models') {
                document.getElementById('section-models').classList.remove('hidden');
                document.getElementById('tab-btn-models').className = 'pb-3 border-b-2 border-blue-500 font-semibold text-blue-400 transition-all';
            } else if (tabId === 'logs') {
                document.getElementById('section-logs').classList.remove('hidden');
                document.getElementById('tab-btn-logs').className = 'pb-3 border-b-2 border-blue-500 font-semibold text-blue-400 transition-all';
            } else if (tabId === 'explorer') {
                document.getElementById('section-explorer').classList.remove('hidden');
                document.getElementById('tab-btn-explorer').className = 'pb-3 border-b-2 border-blue-500 font-semibold text-blue-400 flex items-center space-x-1 transition-all';
                refreshFileTree();
            } else if (tabId === 'eternity') {
                document.getElementById('section-eternity').classList.remove('hidden');
                document.getElementById('tab-btn-eternity').className = 'pb-3 border-b-2 border-blue-500 font-semibold text-blue-400 transition-all';
                fetchEternityProjects();
            }
        }

        async function fetchSystemData() {
            try {
                const res = await fetch('/health');
                if (!res.ok) return;
                const data = await res.json();
                document.getElementById('active-sessions-count').innerText = data.active_sessions || 0;
            } catch (e) {
                console.error(e);
            }
        }

        async function fetchModels() {
            try {
                const res = await fetch('/api/models-status');
                if (!res.ok) return;
                const models = await res.json();
                
                const container = document.getElementById('models-container');
                container.innerHTML = '';
                
                models.forEach(model => {
                    const isRec = model.is_recommended;
                    const isOnline = model.status.includes('ONLINE');
                    
                    const card = document.createElement('div');
                    card.className = `p-4 border rounded-xl bg-gray-950 transition-all ${
                        isRec ? 'border-amber-500/50 glow-amber bg-amber-500/5' : 'border-gray-800 bg-gray-950'
                    }`;
                    
                    card.innerHTML = `
                        <div class="flex items-center justify-between mb-3">
                            <span class="text-xs text-gray-500 code-font truncate max-w-[200px]" title="${model.id}">${model.id}</span>
                            <div class="flex items-center space-x-2">
                                ${isRec ? '<span class="text-[10px] px-1.5 py-0.5 rounded bg-amber-500/10 text-amber-400 border border-amber-500/20 font-bold">★ Recommended</span>' : ''}
                                <span class="h-2 w-2 rounded-full ${isOnline ? 'bg-green-500 animate-pulse' : 'bg-red-500'}"></span>
                                <span class="text-[10px] font-bold ${isOnline ? 'text-green-400' : 'text-red-400'}">${model.status}</span>
                            </div>
                        </div>
                        <h3 class="text-sm font-bold text-gray-200 mb-2 truncate">${model.name}</h3>
                        <div class="flex items-center justify-between text-xs text-gray-400 border-t border-gray-900 pt-2">
                            <span>Type: <strong class="text-gray-300 font-medium">${model.type}</strong></span>
                            <span>Latency: <strong class="text-blue-400 code-font">${model.latency}</strong></span>
                        </div>
                    `;
                    container.appendChild(card);
                });
            } catch (e) {
                console.error(e);
            }
        }

        async function fetchLogs() {
            if (currentTab !== 'logs') return;
            try {
                const res = await fetch('/api/logs');
                if (!res.ok) return;
                const logs = await res.json();
                
                const term = document.getElementById('terminal-content');
                const shouldScroll = term.scrollHeight - term.clientHeight <= term.scrollTop + 50;
                term.innerHTML = logs.map(line => `<div>${line}</div>`).join('');
                if (shouldScroll) {
                    term.scrollTop = term.scrollHeight;
                }
            } catch (e) {
                console.error(e);
            }
        }

        // File Explorer Logic
        async function refreshFileTree() {
            try {
                const res = await fetch('/api/workspace/tree');
                if (!res.ok) return;
                const root = await res.json();
                
                const container = document.getElementById('file-tree');
                container.innerHTML = renderNode(root);
            } catch (e) {
                console.error(e);
            }
        }

        function renderNode(node, depth = 0) {
            const isDir = node.type === 'directory';
            const icon = isDir ? '📁' : '📄';
            const indent = depth * 12;
            
            let html = `
                <div class="flex items-center py-1 px-2 hover:bg-gray-800 rounded cursor-pointer transition-all text-xs" 
                     style="padding-left: ${indent}px"
                     onclick="${isDir ? `toggleDir('${node.path}')` : `openFile('${node.path}')`}">
                    <span class="mr-2">${icon}</span>
                    <span class="truncate ${isDir ? 'text-gray-300 font-medium' : 'text-gray-400'}">${node.name}</span>
                </div>
            `;
            
            if (isDir && node.children && node.children.length > 0) {
                html += `<div id="dir-${node.path.replace(/\\/g, '-').replace(/\\//g, '-')}" class="space-y-0.5">`;
                node.children.forEach(child => {
                    html += renderNode(child, depth + 1);
                });
                html += `</div>`;
            } else if (isDir && (!node.children || node.children.length === 0)) {
                html += `<div class="text-[10px] text-gray-600 italic" style="padding-left: ${indent + 16}px">(empty)</div>`;
            }
            
            return html;
        }

        async function openFile(path) {
            document.getElementById('editor-title').innerText = `📄 ${path}`;
            document.getElementById('editor-content').innerText = "Loading file content...";
            try {
                const res = await fetch(`/api/workspace/file?path=${encodeURIComponent(path)}`);
                if (!res.ok) {
                    document.getElementById('editor-content').innerText = "Error: Failed to fetch file content.";
                    return;
                }
                const data = await res.json();
                document.getElementById('editor-content').innerText = data.content;
            } catch (e) {
                document.getElementById('editor-content').innerText = `Error: ${e.message}`;
            }
        }

        function toggleDir(path) {
            const safeId = `dir-${path.replace(/\\/g, '-').replace(/\\//g, '-')}`;
            const elem = document.getElementById(safeId);
            if (elem) {
                elem.classList.toggle('hidden');
            }
        }

        // Eternity R&D UI Logic
        function openNewProjectModal() {
            document.getElementById('new-project-modal').classList.remove('hidden');
        }

        function closeNewProjectModal() {
            document.getElementById('new-project-modal').classList.add('hidden');
        }

        async function submitNewProject() {
            const name = document.getElementById('proj-name').value.trim();
            const goal = document.getElementById('proj-goal').value.trim();
            const deadline = parseFloat(document.getElementById('proj-deadline').value) || 1.0;
            const priority = document.getElementById('proj-priority').value;

            if (!goal) {
                alert('Please enter a goal statement.');
                return;
            }

            try {
                const res = await fetch('/api/eternity/init', {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
                    body: JSON.stringify({
                        project_name: name,
                        goal: goal,
                        deadline_hours: deadline,
                        priority: priority
                    })
                });
                if (res.ok) {
                    closeNewProjectModal();
                    // Clear fields
                    document.getElementById('proj-name').value = '';
                    document.getElementById('proj-goal').value = '';
                    fetchEternityProjects();
                } else {
                    const data = await res.json();
                    alert(`Failed: ${data.detail || 'Unknown error'}`);
                }
            } catch (e) {
                alert(`Error: ${e.message}`);
            }
        }

        async function toggleActive(projName, isActive) {
            try {
                await fetch('/api/eternity/toggle', {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
                    body: JSON.stringify({ project_name: projName, is_active: isActive })
                });
                fetchEternityProjects();
            } catch (e) {
                console.error(e);
            }
        }

        async function setPriority(projName, priorityVal) {
            try {
                await fetch('/api/eternity/set-priority', {
                    method: 'POST',
                    headers: { 'Content-Type': 'application/json' },
                    body: JSON.stringify({ project_name: projName, priority: priorityVal })
                });
                fetchEternityProjects();
            } catch (e) {
                console.error(e);
            }
        }

        async function fetchEternityProjects() {
            if (currentTab !== 'eternity') return;
            try {
                const res = await fetch('/api/eternity/list');
                if (!res.ok) return;
                const data = await res.json();
                
                const container = document.getElementById('eternity-projects-list');
                if (data.projects.length === 0) {
                    container.innerHTML = `
                        <div class="text-center py-12 border border-dashed border-gray-800 rounded-lg text-gray-500">
                            No active R&D goals configured. Click "New R&D Goal" to launch one!
                        </div>
                    `;
                    return;
                }
                
                container.innerHTML = data.projects.map(p => {
                    const modeColor = p.current_mode === 'build' ? 'from-amber-500/20 to-orange-500/20 text-orange-400 border-orange-500/30' : 'from-green-500/20 to-emerald-500/20 text-green-400 border-green-500/30';
                    const activeBadge = p.is_active ? '<span class="bg-green-500/10 text-green-400 border border-green-500/20 px-2 py-0.5 rounded text-[10px] font-bold">ACTIVE</span>' : '<span class="bg-gray-800 text-gray-500 border border-gray-700 px-2 py-0.5 rounded text-[10px] font-bold">PAUSED</span>';
                    const priorityColor = p.priority === 'supreme' ? 'bg-red-500/10 text-red-400 border-red-500/20' : 'bg-blue-500/10 text-blue-400 border-blue-500/20';
                    
                    return `
                        <div class="bg-gray-950 border border-gray-800 rounded-xl p-6 space-y-4 hover:border-gray-700 transition-all">
                            <div class="flex items-start justify-between">
                                <div class="space-y-1">
                                    <div class="flex items-center space-x-2">
                                        <h3 class="text-lg font-bold text-gray-200 code-font">${p.project_name}</h3>
                                        ${activeBadge}
                                        <span class="px-2 py-0.5 rounded text-[10px] font-bold border ${priorityColor}">${p.priority.toUpperCase()}</span>
                                    </div>
                                    <p class="text-xs text-gray-500">Started on ${new Date(p.created_at).toLocaleString()}</p>
                                </div>
                                <div class="flex items-center space-x-3">
                                    <select onchange="setPriority('${p.project_name}', this.value)" class="bg-black border border-gray-850 rounded px-2.5 py-1 text-xs text-gray-300 focus:outline-none">
                                        <option value="supreme" ${p.priority === 'supreme' ? 'selected' : ''}>Supreme</option>
                                        <option value="low" ${p.priority === 'low' ? 'selected' : ''}>Low</option>
                                    </select>
                                    
                                    <button onclick="toggleActive('${p.project_name}', ${!p.is_active})" class="text-xs px-3.5 py-1 rounded border ${p.is_active ? 'border-red-500/30 text-red-400 bg-red-500/5 hover:bg-red-500/10' : 'border-green-500/30 text-green-400 bg-green-500/5 hover:bg-green-500/10'} font-semibold transition-all">
                                        ${p.is_active ? 'Pause' : 'Resume'}
                                    </button>
                                </div>
                            </div>
                            
                            <div class="space-y-2">
                                <div class="text-sm text-gray-300 font-semibold">Goal Description:</div>
                                <div class="text-sm text-gray-400 bg-black/40 p-3 rounded border border-gray-900 leading-relaxed">${p.goal}</div>
                            </div>
                            
                            <div class="grid grid-cols-2 gap-4 text-xs">
                                <div class="bg-gray-900/50 p-3 rounded border border-gray-850 space-y-1">
                                    <div class="text-gray-500">Mode Status</div>
                                    <div class="font-bold bg-gradient-to-r ${modeColor} bg-clip-text text-transparent">${p.current_mode.toUpperCase()} MODE</div>
                                </div>
                                <div class="bg-gray-900/50 p-3 rounded border border-gray-850 space-y-1">
                                    <div class="text-gray-500">Deadline Countdown</div>
                                    <div class="font-bold text-gray-200 code-font">${p.time_remaining_str || 'Expired'}</div>
                                </div>
                            </div>
                            
                            ${p.latest_brief ? `
                                <div class="bg-blue-950/20 border border-blue-900/30 rounded p-4 text-xs text-blue-300 leading-relaxed space-y-1">
                                    <div class="font-bold text-blue-400 flex items-center space-x-1">
                                        <span>📘 Latest Research Brief Summary</span>
                                    </div>
                                    <div class="text-blue-300 mt-1">${p.latest_brief}</div>
                                </div>
                            ` : ''}
                        </div>
                    `;
                }).join('');
            } catch (e) {
                console.error(e);
            }
        }

        setInterval(fetchSystemData, 3000);
        setInterval(fetchModels, 3000);
        setInterval(fetchLogs, 2000);
        setInterval(fetchEternityProjects, 5000);
        
        fetchSystemData();
        fetchModels();
        fetchLogs();
    </script>
</body>
</html>
"""

# ---------------------------------------------------------------------------
# Concurrency Settings
# ---------------------------------------------------------------------------
completions_semaphore = asyncio.Semaphore(2)


# ---------------------------------------------------------------------------
# SessionStore API (Claude Agent SDK / Claude Code Compatible)
# ---------------------------------------------------------------------------

class SessionAppendRequest(BaseModel):
    project_key: str
    session_id: str
    subpath: Optional[str] = None
    entries: List[Dict[str, Any]]

@app.post("/api/sessions/append")
async def append_session_entries(req: SessionAppendRequest, authorization: str = Header(None)):
    auth(authorization)
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        # Check for mirror_error and log/alert if present
        for e in req.entries:
            if e.get("type") == "system" and e.get("subtype") == "mirror_error":
                log_activity(f"[ALERT] Claude Agent SDK reported mirror_error: {e.get('message')}")
                
        rows = [(req.project_key, req.session_id, req.subpath, json.dumps(e)) for e in req.entries]
        async with db_pool.acquire() as conn:
            await conn.executemany(
                "INSERT INTO agent_session_entries (project_key, session_id, subpath, entry) VALUES ($1, $2, $3, $4)",
                rows
            )
        return {"status": "success"}
    except Exception as e:
        log_activity(f"[SessionStore Error] Failed append: {e}")
        raise HTTPException(status_code=500, detail=str(e))

class SessionLoadRequest(BaseModel):
    project_key: str
    session_id: str
    subpath: Optional[str] = None

@app.post("/api/sessions/load")
async def load_session_entries(req: SessionLoadRequest, authorization: str = Header(None)):
    auth(authorization)
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            rows = await conn.fetch(
                "SELECT entry FROM agent_session_entries WHERE project_key=$1 AND session_id=$2 AND subpath IS NOT DISTINCT FROM $3 ORDER BY id",
                req.project_key, req.session_id, req.subpath
            )
        return {"entries": [json.loads(r["entry"]) for r in rows]}
    except Exception as e:
        log_activity(f"[SessionStore Error] Failed load: {e}")
        raise HTTPException(status_code=500, detail=str(e))

class SessionListRequest(BaseModel):
    project_key: str

@app.post("/api/sessions/list")
async def list_sessions(req: SessionListRequest, authorization: str = Header(None)):
    auth(authorization)
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            rows = await conn.fetch(
                "SELECT DISTINCT session_id FROM agent_session_entries WHERE project_key=$1",
                req.project_key
            )
        return {"sessions": [r["session_id"] for r in rows]}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

class SessionDeleteRequest(BaseModel):
    project_key: str
    session_id: str

@app.post("/api/sessions/delete")
async def delete_session(req: SessionDeleteRequest, authorization: str = Header(None)):
    auth(authorization)
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            await conn.execute(
                "DELETE FROM agent_session_entries WHERE project_key=$1 AND session_id=$2",
                req.project_key, req.session_id
            )
        return {"status": "success"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


from datetime import datetime, timedelta, timezone

class EternityInitRequest(BaseModel):
    project_name: Optional[str] = None
    goal: Optional[str] = None
    problem_statement: Optional[str] = None
    deadline_hours: float
    priority: Optional[str] = "low"

@app.post("/api/eternity/init")
async def init_eternity_system(req: EternityInitRequest):
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        goal_text = req.goal or req.problem_statement or "Build a calculator"
        # Generate slugified project name if not provided
        import re
        slug = req.project_name
        if not slug:
            slug = re.sub(r'[^a-z0-9]+', '-', goal_text.lower()).strip('-')[:30] or "project"
            
        deadline = datetime.now(timezone.utc) + timedelta(hours=req.deadline_hours)
        async with db_pool.acquire() as conn:
            await conn.execute("""
                INSERT INTO eternity_projects (project_name, goal, deadline, current_mode, priority, is_active)
                VALUES ($1, $2, $3, 'build', $4, true)
                ON CONFLICT (project_name) DO UPDATE 
                SET goal = EXCLUDED.goal, deadline = EXCLUDED.deadline, current_mode = 'build', priority = EXCLUDED.priority, is_active = true
            """, slug, goal_text, deadline, req.priority or "low")
        log_activity(f"[Eternity Loop] Project '{slug}' initialized | Goal: '{goal_text}' | Priority: {req.priority}")
        return {"status": "success", "project_name": slug, "deadline": deadline.isoformat()}
    except Exception as e:
        log_activity(f"[Eternity Loop Error] Init failed: {e}")
        raise HTTPException(status_code=500, detail=str(e))

class EternityToggleRequest(BaseModel):
    project_name: str
    is_active: bool

@app.post("/api/eternity/toggle")
async def toggle_project(req: EternityToggleRequest):
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            await conn.execute("UPDATE eternity_projects SET is_active = $1 WHERE project_name = $2", req.is_active, req.project_name)
        log_activity(f"[Eternity Loop] Project '{req.project_name}' is_active set to {req.is_active}")
        return {"status": "success"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

class EternityPriorityRequest(BaseModel):
    project_name: str
    priority: str

@app.post("/api/eternity/set-priority")
async def set_project_priority(req: EternityPriorityRequest):
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        if req.priority not in ["supreme", "low"]:
            raise HTTPException(status_code=400, detail="Invalid priority")
        async with db_pool.acquire() as conn:
            await conn.execute("UPDATE eternity_projects SET priority = $1 WHERE project_name = $2", req.priority, req.project_name)
        log_activity(f"[Eternity Loop] Project '{req.project_name}' priority set to {req.priority}")
        return {"status": "success"}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/eternity/list")
async def list_eternity_projects():
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            rows = await conn.fetch("SELECT project_name, goal, deadline, current_mode, is_active, priority, latest_brief, created_at FROM eternity_projects ORDER BY created_at DESC")
            projects = []
            now = datetime.now(timezone.utc)
            for r in rows:
                deadline = r["deadline"]
                remaining = max(0.0, (deadline - now).total_seconds())
                
                # Format remaining time
                if remaining > 0:
                    h = int(remaining // 3600)
                    m = int((remaining % 3600) // 60)
                    rem_str = f"{h}h {m}m"
                else:
                    rem_str = "Expired"
                    
                projects.append({
                    "project_name": r["project_name"],
                    "goal": r["goal"],
                    "deadline": deadline.isoformat(),
                    "current_mode": r["current_mode"],
                    "is_active": r["is_active"],
                    "priority": r["priority"],
                    "latest_brief": r["latest_brief"],
                    "created_at": r["created_at"].isoformat(),
                    "time_remaining_str": rem_str
                })
            return {"projects": projects}
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/api/eternity/status")
async def get_eternity_status():
    if not db_pool:
        raise HTTPException(status_code=500, detail="Database not connected")
    try:
        async with db_pool.acquire() as conn:
            row = await conn.fetchrow("SELECT project_name, goal, deadline, current_mode, roadmap, latest_brief FROM eternity_projects WHERE is_active = true ORDER BY id DESC LIMIT 1")
            if not row:
                return {"active": False}
                
            deadline = row["deadline"]
            now = datetime.now(timezone.utc)
            remaining = max(0.0, (deadline - now).total_seconds())
            
            return {
                "active": True,
                "project_name": row["project_name"],
                "goal": row["goal"],
                "deadline": deadline.isoformat(),
                "current_mode": row["current_mode"],
                "roadmap": json.loads(row["roadmap"]) if isinstance(row["roadmap"], str) else row["roadmap"],
                "latest_brief": row["latest_brief"],
                "time_remaining_seconds": remaining
            }
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))


class ForgeExecuteRequest(BaseModel):
    task_id: str
    action: str
    prompt: str
    context_rules: Optional[str] = ""
    project_state_md: Optional[str] = ""  # Karpathy CLAUDE.md protocol — shared project context

@app.post("/api/forge/execute")
async def forge_execute(req: ForgeExecuteRequest, authorization: str = Header(None)):
    auth(authorization)
    log_activity(f"[Forge] Received execution request for task {req.task_id}: '{req.prompt[:80]}'")
    
    project_folder = "default"
    if "_" in req.task_id:
        parts = req.task_id.split("_")
        if len(parts) >= 2:
            project_folder = parts[1].replace(" ", "-").lower()
    active_workspace = os.path.join(WORKSPACE_DIR, project_folder)
    os.makedirs(active_workspace, exist_ok=True)
    workspace_token = workspace_var.set(active_workspace)

    try:
        # Load active MCP tools
        mcp_tools, mcp_mapping = await get_active_mcp_tools()
        active_tools = list(TOOLS) + mcp_tools

        # Inject project_state.md into system context if provided (Karpathy protocol)
        system_content = AGENTIC_SYSTEM_PROMPT + f"\nContext rules: {req.context_rules}"
        if req.project_state_md:
            system_content += f"\n\n---\n## Project State Document (read this first)\n{req.project_state_md}"

        messages = [
            {"role": "system", "content": system_content},
            {"role": "user", "content": req.prompt}
        ]
        
        final_messages = list(messages)
        success = False
        summary = ""
        error = None
        
        for round_num in range(MAX_TOOL_ROUNDS + 1):
            await rate_limiter.wait_for_nim()
            response = None
            try:
                response = await nim_client.chat.completions.create(
                    model=RECOMMENDED_MODEL,
                    messages=final_messages,
                    tools=active_tools,
                    tool_choice="auto",
                    timeout=8.0
                )
            except Exception as e:
                log_activity(f"[Forge Fallback] NIM completions failed: {e}. Retrying with Mistral Large...")
                try:
                    if mistral_client:
                        response = await mistral_client.chat.completions.create(
                            model="mistral-large-latest",
                            messages=final_messages,
                            tools=active_tools,
                            tool_choice="auto"
                        )
                    else:
                        raise Exception("Mistral client not initialized")
                except Exception as e2:
                    log_activity(f"[Forge Fallback Error] Mistral completions failed too: {e2}")
                    raise e2
            
            msg = response.choices[0].message
            tool_calls_payload = None
            if msg.tool_calls:
                tool_calls_payload = []
                for tc in msg.tool_calls:
                    tc_dict = {
                        "id": tc.id,
                        "type": tc.type,
                        "function": {
                            "name": tc.function.name,
                            "arguments": tc.function.arguments
                        }
                    }
                    tool_calls_payload.append(tc_dict)
                    
            final_messages.append({
                "role": "assistant",
                "content": msg.content,
                "tool_calls": tool_calls_payload
            })
            
            if not msg.tool_calls:
                success = True
                summary = msg.content or "Task completed."
                break
                
            for tc in msg.tool_calls:
                func_name = tc.function.name
                raw_args_str = tc.function.arguments
                try:
                    func_args = json.loads(raw_args_str)
                except Exception:
                    repaired_str = raw_args_str.strip()
                    if not repaired_str.startswith("{"):
                        repaired_str = "{" + repaired_str
                    if not repaired_str.endswith("}"):
                        repaired_str = repaired_str + "}"
                    try:
                        func_args = json.loads(repaired_str)
                    except Exception:
                        func_args = {}
                result = await execute_tool(func_name, func_args, mcp_mapping)
                
                final_messages.append({
                    "role": "tool",
                    "tool_call_id": tc.id,
                    "content": result
                })
        else:
            error = "Max tool call rounds exceeded."
            
        if success:
            return {
                "status": "success",
                "summary": summary,
                "error": None
            }
        else:
            return {
                "status": "error",
                "error": error or "Task execution failed."
            }
            
    except Exception as e:
        log_activity(f"[Forge Error] Task execution failed: {e}")
        return {
            "status": "error",
            "error": str(e)
        }
    finally:
        workspace_var.reset(workspace_token)

async def db_heartbeat_loop():
    log_activity("Database Heartbeat task started")
    while True:
        try:
            if db_pool:
                async with db_pool.acquire() as conn:
                    await conn.execute("SELECT 1")
                    log_activity("[Heartbeat] Pinged Aiven PostgreSQL successfully")
        except Exception as e:
            log_activity(f"[Heartbeat Warning] Failed to ping database: {e}")
        await asyncio.sleep(240) # Every 4 minutes


import httpx

# ---------------------------------------------------------------------------
# Inter-Space URLs — The 6-Space Topology
# Space 2 (self)  = shyota/claude-code-backend   — Cerebrum (Orchestrator)
# Space 3         = augment17/claude-code-backend — Forge (Coder)
# Space 4         = shyota/mcp-cloud-host         — Library (Research + MCP)
# Space 5 (Vault) = Space 3 also handles vault push since agent-worker-loop
#                   HF slot is not provisioned. SPACE5_URL = SPACE3_URL.
# Space 6         = augment17/mcp-cloud-host      — Sandbox (UI Tester)
# ---------------------------------------------------------------------------
SPACE3_URL = os.environ.get("SPACE3_URL", "https://augment17-claude-code-backend.hf.space")
SPACE4_URL = os.environ.get("SPACE4_URL", "https://shyota-mcp-cloud-host.hf.space")
SPACE5_URL = os.environ.get("SPACE5_URL", "https://augment17-agent-worker-loop-v2.hf.space")  # Dedicated Vault container
SPACE6_URL = os.environ.get("SPACE6_URL", "https://augment17-mcp-cloud-host.hf.space")

async def get_active_projects():
    async with db_pool.acquire() as conn:
        return await conn.fetch("SELECT project_name, goal, deadline, current_mode, priority FROM eternity_projects WHERE is_active = true ORDER BY id DESC")

async def update_db_mode(project_name: str, mode: str):
    async with db_pool.acquire() as conn:
        await conn.execute("UPDATE eternity_projects SET current_mode = $1 WHERE project_name = $2", mode, project_name)

async def update_db_brief(project_name: str, brief: str):
    async with db_pool.acquire() as conn:
        await conn.execute("UPDATE eternity_projects SET latest_brief = $1 WHERE project_name = $2", brief, project_name)

async def update_db_roadmap(project_name: str, roadmap: list):
    async with db_pool.acquire() as conn:
        await conn.execute("UPDATE eternity_projects SET roadmap = $1 WHERE project_name = $2", json.dumps(roadmap), project_name)

# Async HTTP POST — replaces blocking urllib.request so the event loop never freezes
async def apost_json(url: str, payload: dict, timeout: float = 120.0) -> dict:
    headers = {
        "Authorization": f"Bearer {BACKEND_API_KEY}",
        "Content-Type": "application/json"
    }
    try:
        async with httpx.AsyncClient(timeout=timeout) as client:
            r = await client.post(url, json=payload, headers=headers)
            r.raise_for_status()
            return r.json()
    except Exception as e:
        log_activity(f"[HTTP Error] POST to {url} failed: {e}")
        return {"status": "error", "error": str(e)}

# ---------------------------------------------------------------------------
# Karpathy project_state.md — Shared Context Document
# Each space receives this instead of bare prompts so agents always know
# what phase they are in, what was built before, and what the research says.
# ---------------------------------------------------------------------------
def build_project_state_md(project_name: str, goal: str, mode: str,
                            task_prompt: str, brief: str = "",
                            prior_summary: str = "") -> str:
    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%MZ")
    return f"""# Project State: {project_name}
_Generated by Space 2 (Cerebrum) at {ts}_

## Current Objective
**Mode:** {mode.upper()}
**Goal:** {goal}

## Instructions for Space 3 (The Forge)
{task_prompt}

## Research Brief (from Space 4 — The Library)
{brief if brief else '_No research brief yet._'}

## Prior Work Summary
{prior_summary if prior_summary else '_First cycle._'}

## Context Rules
- Write deterministic, testable code.
- All files go in /tmp/workspace/.
- Return a 3-line summary: status, file changed, test result.
"""

async def execute_build_cycle(project_name: str, goal: str):
    log_activity(f"[Build Mode] Initiating build cycle for project '{project_name}' (goal: '{goal}')")
    
    # Track project state in our in-process graph DB
    helix_db.upsert_project(project_name, goal, "build", "low")
    
    await rate_limiter.wait_for_nim()
    plan_prompt = (
        f"We are building: '{goal}'. "
        "Write ONE JSON instruction for Space 3 (The Forge) to code the next milestone. "
        "Respond ONLY with JSON: {\"prompt\": \"task description\", \"context_rules\": \"rules\"}"
    )
    raw = ""
    try:
        res = await nim_client.chat.completions.create(
            model=RECOMMENDED_MODEL,
            messages=[{"role": "user", "content": plan_prompt}],
            max_tokens=1024,
            timeout=8.0
        )
        raw = res.choices[0].message.content.strip()
    except Exception as e:
        log_activity(f"[Model Fallback] NIM failed ({e}). Retrying with Mistral Large...")
        try:
            if mistral_client:
                res = await mistral_client.chat.completions.create(
                    model="mistral-large-latest",
                    messages=[{"role": "user", "content": plan_prompt}],
                    max_tokens=1024
                )
                raw = res.choices[0].message.content.strip()
            else:
                raise Exception("Mistral client not initialized")
        except Exception as e2:
            log_activity(f"[Model Fallback Error] Mistral failed too: {e2}")
            return

    try:
        if raw.startswith("```"):
            raw = raw.split("```")[1].lstrip("json").strip()
        task = json.loads(raw)
        task_prompt = task.get("prompt", "")
        context_rules = task.get("context_rules", "")
        if isinstance(context_rules, (dict, list)):
            context_rules = json.dumps(context_rules)
    except Exception as e:
        log_activity(f"[Build Mode Error] Plan JSON parsing failed: {e} | Raw: {raw}")
        return

    # Use the context-locked Bell Curve prompt builder (enforces max 2500 tokens)
    playbook_file = f"space3-forge/debugging/{project_name}_playbook.md"
    state_md = brain.build_apex_prompt(
        project_name=project_name,
        goal=goal,
        mode="build",
        task_instruction=task_prompt,
        research_topic=project_name,
        current_file=""
    )

    log_activity(f"[Build Mode] Dispatching to Space 3 (Forge) — project '{project_name}': '{task_prompt[:80]}'")
    forge_res = await apost_json(f"{SPACE3_URL}/api/forge/execute", {
        "task_id": f"build_{project_name}_{int(time.time())}",
        "action": "execute_code",
        "prompt": task_prompt,
        "context_rules": context_rules,
        "project_state_md": state_md
    })

    if forge_res.get("status") == "success":
        summary = forge_res.get("summary", "")
        log_activity(f"[Build Mode] Space 3 (Forge) SUCCESS for '{project_name}': {summary}")

        log_activity(f"[Build Mode] Triggering Space 6 (Sandbox) UI verification for '{project_name}'...")
        test_res = await apost_json(f"{SPACE6_URL}/api/sandbox/test", {
            "test_cmd": "verify_ui",
            "url": SPACE3_URL,
            "project_name": project_name
        }, timeout=30.0)
        
        verdict = test_res.get("verdict", "UNKNOWN")
        reason = test_res.get("reason", "")
        log_activity(f"[Build Mode] Space 6 (Sandbox) Verdict: {verdict} — {reason}")

        # --- ACE: Reflect & Curate ---
        # Runs the feedback loop to update playbook rules on failure, or record patterns on success
        ace_verdict = await context_engine.reflect_and_curate(
            project_name=project_name,
            task_prompt=task_prompt,
            result_summary=summary,
            verdict=verdict,
            reason=reason
        )
        log_activity(f"[Build Mode] ACE Reflection Result: {ace_verdict}")

        # Update in-process graph DB cycle and state
        helix_db.record_cycle(project_name, summary, verdict)

        # Log event directly to the local Second Brain git repo
        brain.append(
            "space2-cerebrum/loop_log.md",
            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Build Success | Project: {project_name} | Summary: {summary[:80]} | Verdict: {verdict}",
            f"[Brain Log] Record build cycle for {project_name}"
        )

        log_activity("[Build Mode] Triggering Space 5 (Vault) backup commit...")
        vault_res = await apost_json(f"{SPACE5_URL}/api/vault/push", {
            "project_name": project_name,
            "summary": summary
        }, timeout=30.0)
        if vault_res.get("status") == "error":
            log_activity("[Build Mode] Space 5 (Vault) failed. Retrying backup via Space 3 (Forge) fallback...")
            vault_res = await apost_json(f"{SPACE3_URL}/api/vault/push", {
                "project_name": project_name,
                "summary": summary
            }, timeout=30.0)
        log_activity(f"[Build Mode] Vault backup status: {vault_res.get('status', 'unknown')}")
    else:
        log_activity(f"[Build Mode Warning] Space 3 (Forge) failure for '{project_name}': {forge_res.get('error')}")


async def execute_eternity_cycle(project_name: str, goal: str):
    log_activity(f"[Eternity Mode] R&D cycle starting for project '{project_name}' (goal: '{goal}')")

    # Track project state in our in-process graph DB
    helix_db.upsert_project(project_name, goal, "eternity", "low")

    # Step 1: Ask Space 4 (Library) for research
    log_activity(f"[Eternity Mode] Querying Space 4 (Library) for '{project_name}'...")
    research_res = await apost_json(f"{SPACE4_URL}/api/research", {
        "query": f"novel algorithms, optimizations, and next features for: {goal}"
    })
    brief = research_res.get("brief", "No new features found.")
    await update_db_brief(project_name, brief)
    log_activity(f"[Eternity Mode] Space 4 (Library) brief received ({len(brief)} chars)")

    # Save research brief to Second Brain wiki on GitHub
    research_file = f"space4-library/research/{project_name.replace(' ', '-')}.md"
    brain.write(
        research_file,
        f"# Research Brief: {project_name}\n\n{brief}",
        f"[Brain Research] Save brief for {project_name}"
    )

    # Step 2: Ask NIM to plan next feature based on research
    await rate_limiter.wait_for_nim()
    plan_prompt = (
        f"Goal: '{goal}'.\nResearch Brief:\n{brief[:2000]}\n\n"
        "Based on this research, plan the SINGLE most impactful next feature to implement. "
        "Respond ONLY with JSON: {\"prompt\": \"task\", \"context_rules\": \"rules\"}"
    )
    raw = ""
    try:
        res = await nim_client.chat.completions.create(
            model=RECOMMENDED_MODEL,
            messages=[{"role": "user", "content": plan_prompt}],
            max_tokens=1024,
            timeout=8.0
        )
        raw = res.choices[0].message.content.strip()
    except Exception as e:
        log_activity(f"[Model Fallback] NIM failed ({e}). Retrying with Mistral Large...")
        try:
            if mistral_client:
                res = await mistral_client.chat.completions.create(
                    model="mistral-large-latest",
                    messages=[{"role": "user", "content": plan_prompt}],
                    max_tokens=1024
                )
                raw = res.choices[0].message.content.strip()
            else:
                raise Exception("Mistral client not initialized")
        except Exception as e2:
            log_activity(f"[Model Fallback Error] Mistral failed too: {e2}")
            return

    try:
        if raw.startswith("```"):
            raw = raw.split("```")[1].lstrip("json").strip()
        task = json.loads(raw)
        task_prompt = task.get("prompt", "")
        context_rules = task.get("context_rules", "")
        if isinstance(context_rules, (dict, list)):
            context_rules = json.dumps(context_rules)
    except Exception as e:
        log_activity(f"[Eternity Mode Error] Plan JSON parsing failed: {e} | Raw: {raw}")
        return

    # Step 3: Build Karpathy context-locked prompt (Apex)
    state_md = brain.build_apex_prompt(
        project_name=project_name,
        goal=goal,
        mode="eternity",
        task_instruction=task_prompt,
        research_topic=project_name
    )

    # Step 4: Dispatch to Space 3 (Forge)
    log_activity(f"[Eternity Mode] Dispatching to Space 3 (Forge): '{task_prompt[:80]}'")
    forge_res = await apost_json(f"{SPACE3_URL}/api/forge/execute", {
        "task_id": f"eternity_{project_name}_{int(time.time())}",
        "action": "execute_code",
        "prompt": task_prompt,
        "context_rules": context_rules,
        "project_state_md": state_md
    })

    if forge_res.get("status") == "success":
        summary = forge_res.get("summary", "")
        log_activity(f"[Eternity Mode] Space 3 (Forge) SUCCESS: {summary}")

        # Update graph DB cycle
        helix_db.record_cycle(project_name, summary, "PASS")

        # Log event directly to the local Second Brain git repo
        brain.append(
            "space2-cerebrum/loop_log.md",
            f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] Eternity Cycle Success | Project: {project_name} | Summary: {summary[:80]}",
            f"[Brain Log] Record eternity cycle for {project_name}"
        )

        # Trigger vault backup after each successful eternity cycle
        vault_res = await apost_json(f"{SPACE5_URL}/api/vault/push", {
            "project_name": project_name,
            "summary": summary
        }, timeout=30.0)
        if vault_res.get("status") == "error":
            log_activity("[Eternity Mode] Space 5 (Vault) failed. Retrying backup via Space 3 (Forge) fallback...")
            vault_res = await apost_json(f"{SPACE3_URL}/api/vault/push", {
                "project_name": project_name,
                "summary": summary
            }, timeout=30.0)
        log_activity(f"[Eternity Mode] Vault backup status: {vault_res.get('status', 'unknown')}")
    else:
        log_activity(f"[Eternity Mode Warning] Space 3 (Forge) failure: {forge_res.get('error')}") 


last_run_times = {}  # project_name -> last execution timestamp
IDLE_TASK_INTERVAL = 1800  # Run idle tasks every 30 min when no active projects
_last_idle_run = 0.0

async def execute_idle_worker_cycle():
    """Idle Worker Protocol — runs when no active projects exist.
    Dispatches self-improvement tasks to Space 3 (Forge) to keep
    all containers warm and utilise free compute."""
    log_activity("[Idle Worker] No active projects. Dispatching background optimisation task to Space 3...")
    idle_tasks = [
        "Scan /tmp/workspace for any Python or JS files. For each file found, "
        "check if a corresponding test file exists. If not, write one basic unit test and run it. "
        "Return a summary of files scanned and tests added.",
        "Scan /tmp/workspace for duplicate function definitions across files. "
        "Report any found and refactor the worst one to a shared utility module.",
        "Run a syntax check on all files in /tmp/workspace and fix any issues found.",
    ]
    import random
    chosen = random.choice(idle_tasks)
    state_md = build_project_state_md(
        project_name="idle-maintenance", goal="system self-improvement",
        mode="idle", task_prompt=chosen
    )
    res = await apost_json(f"{SPACE3_URL}/api/forge/execute", {
        "task_id": f"idle_{int(time.time())}",
        "action": "execute_code",
        "prompt": chosen,
        "context_rules": "Be concise. Fix only what is necessary. Return a 3-line summary.",
        "project_state_md": state_md
    }, timeout=60.0)
    log_activity(f"[Idle Worker] Space 3 response: {res.get('status')} — {str(res.get('summary', ''))[:120]}")

async def run_eternity_loop():
    global _last_idle_run
    log_activity("[Eternity Loop] Async orchestration task started on FastAPI main loop.")
    loop_counter = 0
    while True:
        try:
            if not db_pool:
                await asyncio.sleep(10)
                continue

            projects = await get_active_projects()

            if not projects:
                # --- Idle Worker Protocol ---
                now_ts = time.time()
                if now_ts - _last_idle_run >= IDLE_TASK_INTERVAL:
                    _last_idle_run = now_ts
                    await execute_idle_worker_cycle()
                await asyncio.sleep(60)
                continue

            loop_counter += 1
            for p in projects:
                name = p["project_name"]
                goal = p["goal"]
                deadline = p["deadline"]
                current_mode = p["current_mode"]
                priority = p["priority"]

                # Priority gating: "low" skips 3 of every 4 cycles to conserve RPM
                if priority == "low" and (loop_counter % 4) != 0:
                    log_activity(f"[Eternity Loop] Skipping low-priority '{name}' this cycle ({loop_counter})")
                    continue

                now = datetime.now(timezone.utc)
                if current_mode == "build" and now >= deadline:
                    await update_db_mode(name, "eternity")
                    current_mode = "eternity"
                    log_activity(f"[Eternity Loop] Project '{name}' deadline passed — transitioned to Eternity R&D Mode.")

                if current_mode == "build":
                    await execute_build_cycle(name, goal)
                else:
                    last_run = last_run_times.get(name, 0.0)
                    now_ts = time.time()
                    interval = int(os.environ.get("ETERNITY_LOOP_INTERVAL", "3600"))
                    if now_ts - last_run < interval:
                        remaining_min = int((interval - (now_ts - last_run)) / 60)
                        log_activity(f"[Eternity Loop] Project '{name}' sleeping — next R&D in {remaining_min}m")
                        continue
                    last_run_times[name] = now_ts
                    await execute_eternity_cycle(name, goal)

            # Base check interval: 5 minutes
            await asyncio.sleep(300)

        except Exception as e:
            log_activity(f"[Eternity Loop Error] Unhandled exception: {e}")
            await asyncio.sleep(60)


@app.get("/", response_class=HTMLResponse)
async def dashboard():
    return HTMLResponse(content=DASHBOARD_HTML)


# ---------------------------------------------------------------------------
# Vault Push Endpoint (Space 5 co-hosted here on Space 3)
# Space 2 (Cerebrum) calls this after every successful build/eternity cycle.
# It zips /tmp/workspace and logs the backup event.
# A real git push would require SSH credentials injected via HF secrets.
# ---------------------------------------------------------------------------
class VaultPushRequest(BaseModel):
    project_name: Optional[str] = "unknown"
    summary: Optional[str] = ""

@app.post("/api/vault/push")
async def vault_push(req: VaultPushRequest, authorization: str = Header(None)):
    auth(authorization)
    workspace = "/tmp/workspace"
    archive = "/tmp/vault_snapshot.zip"
    try:
        if os.path.exists(workspace) and os.listdir(workspace):
            proc = await asyncio.create_subprocess_exec(
                "zip", "-r", archive, workspace,
                stdout=asyncio.subprocess.DEVNULL,
                stderr=asyncio.subprocess.DEVNULL
            )
            await asyncio.wait_for(proc.communicate(), timeout=30)
            size_kb = int(os.path.getsize(archive) / 1024) if os.path.exists(archive) else 0
            log_activity(f"[Vault] Snapshot created for '{req.project_name}': {size_kb}KB | {req.summary[:80]}")
            return {"status": "success", "archive_kb": size_kb, "project_name": req.project_name}
        else:
            log_activity(f"[Vault] Workspace empty — no snapshot for '{req.project_name}'")
            return {"status": "skipped", "reason": "workspace empty"}
    except Exception as e:
        log_activity(f"[Vault Error] Snapshot failed for '{req.project_name}': {e}")
        return {"status": "error", "error": str(e)}




@app.get("/api/logs")
async def get_logs():
    return list(activity_logs)


@app.get("/api/metrics")
async def metrics():
    """Exposes live CPU/RAM and HelixStateDB graph stats for space monitoring."""
    vm = watchdog.get_metrics() if hasattr(watchdog, "get_metrics") else get_metrics()
    return {
        "status": "success",
        "timestamp": time.time(),
        "space_resources": vm,
        "graph_database": helix_db.dump() if hasattr(helix_db, "dump") else {}
    }


@app.get("/api/models-status")
async def get_models_status():
    status_list = []
    for model_id, display_name in ALL_MODELS.items():
        status_info = MODEL_STATUSES.get(model_id, {"status": "ONLINE (Unchecked)", "latency": "N/A"})
        is_rec = model_id == RECOMMENDED_MODEL
        is_agentic = model_id in TOOL_CAPABLE_MODELS
        status_list.append({
            "id": model_id,
            "name": display_name,
            "status": status_info["status"],
            "latency": status_info["latency"],
            "is_recommended": is_rec,
            "type": "Agentic (Tools)" if is_agentic else "Chat Only",
        })
    # Sort: Recommended first, then Agentic, then Chat
    status_list.sort(key=lambda m: (not m["is_recommended"], m["type"] != "Agentic (Tools)", m["name"]))
    return status_list


import shutil
import threading
import signal
from fastapi.responses import FileResponse

@app.get("/api/backup/download")
async def download_backup(authorization: str = Header(None)):
    auth(authorization)
    snapshot_dir = "/tmp/workspace_snapshot"
    archive_base = "/tmp/workspace_backup_download"
    archive_zip = archive_base + ".zip"
    
    # Clean up old files/folders
    for path in [snapshot_dir, archive_zip]:
        if os.path.exists(path):
            try:
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
            except Exception:
                pass
                
    try:
        # 1. Atomic-like snapshot copy (ignoring temporary files)
        shutil.copytree(WORKSPACE_DIR, snapshot_dir, symlinks=True, ignore=shutil.ignore_patterns('.git', 'node_modules', '.next'))
        # 2. Archive the snapshot folder to disk to prevent OOM memory spike
        shutil.make_archive(archive_base, 'zip', snapshot_dir)
        # 3. Clean up the snapshot directory immediately
        shutil.rmtree(snapshot_dir)
        
        if not os.path.exists(archive_zip):
            raise HTTPException(status_code=500, detail="Failed to create zip archive")
        return FileResponse(archive_zip, media_type="application/zip", filename="workspace_backup.zip")
    except Exception as e:
        if os.path.exists(snapshot_dir):
            shutil.rmtree(snapshot_dir)
        raise HTTPException(status_code=500, detail=str(e))


class SetModelRequest(BaseModel):
    model: str

@app.post("/api/settings/model")
async def set_recommended_model(req: SetModelRequest, authorization: str = Header(None)):
    auth(authorization)
    global RECOMMENDED_MODEL
    if req.model not in TOOL_CAPABLE_MODELS:
        raise HTTPException(status_code=400, detail="Invalid or unsupported agent model")
    RECOMMENDED_MODEL = req.model
    log_activity(f"[Settings] Recommended model manually updated to: {RECOMMENDED_MODEL}")
    return {"status": "ok", "recommended_model": RECOMMENDED_MODEL}


@app.get("/health")
async def health():
    return {
        "status": "ok",
        "workspace": WORKSPACE_DIR,
        "workspace_exists": Path(WORKSPACE_DIR).exists(),
        "db_connected": db_pool is not None,
        "models_count": len(ALL_MODELS),
        "recommended_model": RECOMMENDED_MODEL,
        "active_sessions": len(ACTIVE_SESSIONS),
    }


# ---------------------------------------------------------------------------
# Watchdog Daemon for Claude Code Subprocesses (Orphan Reaper)
# ---------------------------------------------------------------------------

def run_watchdog():
    log_activity("System Watchdog Daemon started (PPID-based Orphan detection)")
    while True:
        try:
            import psutil
            for proc in psutil.process_iter(['pid', 'ppid', 'name', 'cmdline', 'status']):
                try:
                    cmd = " ".join(proc.info['cmdline'] or [])
                    # Match the CLI binary (looks like claude-code or anthropic CLI wrapper)
                    if "claude" in cmd.lower() or "anthropic" in cmd.lower():
                        ppid = proc.info['ppid']
                        pid = proc.info['pid']
                        
                        # Is the parent still alive and not a zombie?
                        parent_exists = False
                        if ppid != 1:  # Orphaned processes get reparented to PID 1 in Linux
                            try:
                                parent_proc = psutil.Process(ppid)
                                if parent_proc.is_running() and parent_proc.status() != psutil.STATUS_ZOMBIE:
                                    parent_exists = True
                            except psutil.NoSuchProcess:
                                pass
                        
                        if not parent_exists:
                            log_activity(f"[Watchdog SIGKILL] Reaping orphaned Claude Code process PID {pid} (PPID {ppid})")
                            proc.terminate()
                            time.sleep(2)
                            if proc.is_running():
                                proc.kill()
                except Exception:
                    continue
        except ImportError:
            # Fallback zero-dependency shell parser using /proc
            try:
                # Find all processes and examine their parent PID
                out = subprocess.check_output("ps -o pid,ppid,args | grep -E 'claude|anthropic' | grep -v grep", shell=True, text=True)
                for line in out.strip().split("\n"):
                    parts = line.strip().split(None, 2)
                    if len(parts) >= 2:
                        pid = int(parts[0])
                        ppid = int(parts[1])
                        
                        # Check if parent pid exists/is alive
                        parent_exists = False
                        if ppid != 1:
                            # Check /proc/[ppid] directory
                            if os.path.exists(f"/proc/{ppid}"):
                                parent_exists = True
                                
                        if not parent_exists:
                            log_activity(f"[Watchdog SIGKILL Fallback] Reaping orphaned process PID {pid} (PPID {ppid})")
                            try:
                                os.kill(pid, signal.SIGTERM)
                                time.sleep(2)
                                os.kill(pid, signal.SIGKILL)
                            except Exception:
                                pass
            except Exception:
                pass
        except Exception as e:
            log_activity(f"[Watchdog Error] {e}")
        time.sleep(60)


async def db_heartbeat_loop():
    log_activity("Database Heartbeat task started")
    while True:
        try:
            if db_pool:
                async with db_pool.acquire() as conn:
                    await conn.execute("SELECT 1")
                    log_activity("[Heartbeat] Pinged Aiven PostgreSQL successfully")
        except Exception as e:
            log_activity(f"[Heartbeat Warning] Failed to ping database: {e}")
        await asyncio.sleep(240) # Every 4 minutes


def run_backup_loop():
    if os.environ.get("SPACE_NAME", "space2-cerebrum") == "space2-cerebrum":
        log_activity("Local Git Backup Loop disabled on Cerebrum (Space 2)")
        return
    log_activity("Local Git Backup Loop started")
    while True:
        # Wait 5 minutes between runs
        time.sleep(300)
        if not BACKUP_GIT_REPO:
            continue
        try:
            log_activity("[Backup] Starting local workspace backup...")
            # 1. Clean local backup directory
            backup_local_dir = "/tmp/git_backup_repo"
            if os.path.exists(backup_local_dir):
                shutil.rmtree(backup_local_dir)
            os.makedirs(backup_local_dir, exist_ok=True)
            
            # 2. Point-in-time snapshot copy
            snapshot_dir = "/tmp/workspace_snapshot"
            if os.path.exists(snapshot_dir):
                shutil.rmtree(snapshot_dir)
            shutil.copytree(WORKSPACE_DIR, snapshot_dir, symlinks=True, ignore=shutil.ignore_patterns('.git', 'node_modules', '.next'))
            
            # 3. Zip snapshot
            archive_base = "/tmp/workspace_backup_download"
            archive_zip = archive_base + ".zip"
            if os.path.exists(archive_zip):
                os.unlink(archive_zip)
            shutil.make_archive(archive_base, 'zip', snapshot_dir)
            shutil.rmtree(snapshot_dir)
            
            # 4. Handle Split if needed
            target_dest = os.path.join(backup_local_dir, "workspace_backup.zip")
            zip_size = os.path.getsize(archive_zip)
            max_part_size = 50 * 1024 * 1024 # 50MB
            if zip_size > max_part_size:
                subprocess.run(f"split -b 50M {archive_zip} {target_dest}.part", shell=True)
            else:
                shutil.copy(archive_zip, target_dest)
            os.unlink(archive_zip)
            
            # 5. Git commit and force push
            subprocess.run("git init", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run("git config user.name 'Backup Agent'", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run("git config user.email 'backup@agent.internal'", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run(f"git remote add origin {BACKUP_GIT_REPO}", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run("git checkout -b main", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run("git add -A", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            subprocess.run('git commit -m "Auto-backup: ' + time.strftime("%Y-%m-%d %H:%M:%S") + '"', shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            
            result = subprocess.run("git push origin main --force", shell=True, cwd=backup_local_dir, stdout=subprocess.DEVNULL)
            if result.returncode == 0:
                log_activity("[Backup] Sync completed successfully (git history purged)")
            else:
                log_activity("[Backup Error] Git push failed")
        except Exception as e:
            log_activity(f"[Backup Error] {e}")


async def db_cleanup_loop():
    log_activity("Database Retention Cleanup task started")
    while True:
        try:
            if db_pool:
                async with db_pool.acquire() as conn:
                    result = await conn.execute(
                        "DELETE FROM agent_session_entries WHERE created_at < NOW() - INTERVAL '30 days'"
                    )
                    log_activity(f"[Cleanup] Nightly retention sweep complete. Status: {result}")
        except Exception as e:
            log_activity(f"[Cleanup Warning] Failed to run retention cleanup: {e}")
        await asyncio.sleep(86400) # Every 24 hours


async def wiki_compactor_loop():
    log_activity("[Compactor] Wiki auto-compactor background task started.")
    while True:
        try:
            await context_engine.compact_wiki()
        except Exception as e:
            log_activity(f"[Compactor Error] {e}")
        await asyncio.sleep(86400) # Every 24 hours


@app.on_event("startup")
async def startup_event():
    # Initialize the Ultimate Brain components
    try:
        log_activity("[Startup] Initialising Second Brain (git clone)...")
        brain.initialize()
        log_activity("[Startup] Second Brain cache ready.")
        
        log_activity("[Startup] Warming up local SwarmLLM (Qwen2.5-1.5B)...")
        asyncio.create_task(swarm.warm_up())
    except Exception as e:
        log_activity(f"[Startup Error] Brain init failed: {e}")

    # Start the Resource Watchdog background task
    asyncio.create_task(watchdog.run())
    # Start the Git-Sync background task
    asyncio.create_task(brain.background_sync())
    # Start the Wiki Auto-Compaction task
    asyncio.create_task(wiki_compactor_loop())

    # Start the local backup loop thread
    threading.Thread(target=run_backup_loop, daemon=True).start()
    # Start the eternity R&D loop as an async task on the main event loop
    asyncio.create_task(run_eternity_loop())
    # Start the db keep-alive loop on FastAPI event loop
    asyncio.create_task(db_heartbeat_loop())
    # Start the db nightly retention cleanup loop
    asyncio.create_task(db_cleanup_loop())



# ---------------------------------------------------------------------------
# Entrypoint
# ---------------------------------------------------------------------------

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=7860)