Delete app.py

app.py

@@ -1,192 +0,0 @@
-from __future__ import annotations
-import json
-import os
-import socket
-import subprocess
-import tarfile
-import time
-import urllib.request
-import zipfile
-
-from fastapi import Request
-from fastapi.responses import StreamingResponse
-from gradio import Server
-import httpx
-
-app = Server()
-
-try:
-    import spaces
-    HAS_SPACES = True
-except ImportError:
-    HAS_SPACES = False
-
-@spaces.GPU if HAS_SPACES else lambda f: f
-def whygpu():
-    pass
-
-# Guard so llama-server (and therefore the model weights) can never be
-# loaded into memory more than once, even if start_llama_server() ends up
-# being called again (module re-import, a reload, a duplicate call, etc.).
-_llama_server_started = False
-
-
-def _is_llama_server_up(timeout: float = 1.0) -> bool:
-    """Return True if something is already listening on the llama-server port."""
-    try:
-        with socket.create_connection(("127.0.0.1", 8000), timeout=timeout):
-            return True
-    except OSError:
-        return False
-
-
-def build_latest_llama_server():
-    binary_path = os.path.abspath("llama.cpp/build/bin/llama-server")
-    if os.path.exists(binary_path):
-        return binary_path
-
-    print("Building latest llama.cpp from master source...")
-
-    if not os.path.exists("llama.cpp"):
-        subprocess.run(["git", "clone", "https://github.com/unslothai/llama.cpp.git"], check=True)
-    else:
-        subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
-
-    subprocess.run(["git", "fetch", "origin", "pull/144/head:mtp"], cwd="llama.cpp", check=True)
-    subprocess.run(["git", "checkout", "mtp"], cwd="llama.cpp", check=True)
-
-    env = os.environ.copy()
-    if "/usr/local/cuda/bin" not in env.get("PATH", ""):
-        env["PATH"] = f"/usr/local/cuda/bin:{env.get('PATH', '')}"
-
-    cmake_cmd = [
-        "cmake",
-        "-B",
-        "build",
-        "-DCMAKE_BUILD_TYPE=Release",
-        "-DGGML_NATIVE=ON",
-        "-DGGML_OPENMP=ON",
-        "-DGGML_AVX512=ON",
-    ]
-
-    subprocess.run(cmake_cmd, cwd="llama.cpp", env=env, check=True)
-    subprocess.run(
-        ["cmake", "--build", "build", "--config", "Release", "-j", str(os.cpu_count() or 4), "--target", "llama-server"],
-        cwd="llama.cpp",
-        env=env,
-        check=True
-    )
-
-    if not os.path.exists(binary_path):
-        raise RuntimeError("Failed to build llama-server binary!")
-
-    return binary_path
-
-
-def start_llama_server():
-    global _llama_server_started
-
-    # 1) Already started earlier in this same Python process -> do nothing.
-    if _llama_server_started:
-        print("start_llama_server() called again in this process - already running, skipping.")
-        return None
-
-    # 2) A llama-server (from an earlier run / another worker) is already
-    #    listening on the port -> don't spin up a second copy of the model.
-    if _is_llama_server_up():
-        print("Port 8000 is already serving llama-server - skipping duplicate startup.")
-        _llama_server_started = True
-        return None
-
-    binary_path = build_latest_llama_server()
-
-    from huggingface_hub import hf_hub_download
-
-    model_path = hf_hub_download(
-        repo_id="bartowski/moonshotai_Kimi-Linear-48B-A3B-Instruct-GGUF",
-        filename="moonshotai_Kimi-Linear-48B-A3B-Instruct-Q4_K_M.gguf",
-    )
-
-    binary_dir = os.path.dirname(os.path.abspath(binary_path))
-    env = os.environ.copy()
-    env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
-
-    cmd = [
-        binary_path,
-        "-m", model_path,
-        "--port", "8000",
-        "--host", "127.0.0.1",
-        "-t", "16",
-        "-tb", "16",
-        "-fa", "on",
-        "--parallel", "1",
-        "--load-mode", "dio",
-        "--cache-type-k", "q4_0",
-        "--cache-type-v", "q4_0",
-        "--jinja",
-        "--temp", "1.0",
-        "--top-p", "0.95",
-        "--top-k", "20",
-        "--min-p", "0.0",
-        "--presence-penalty", "0.0",
-        "--repeat-penalty", "1.0",
-    ]
-
-    process = subprocess.Popen(cmd, env=env)
-    _llama_server_started = True
-    time.sleep(5)
-    return process
-
-
-@app.middleware("http")
-async def proxy_middleware(request: Request, call_next):
-    path = request.url.path
-    if path.startswith("/gradio_api"):
-        return await call_next(request)
-
-    url = f"http://127.0.0.1:8000{path}"
-
-    headers = {
-        k: v for k, v in request.headers.items()
-        if k.lower() not in ("host", "content-length", "accept-encoding")
-    }
-    body = await request.body()
-
-    client = httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=120.0))
-    req = client.build_request(
-        method=request.method,
-        url=url,
-        headers=headers,
-        params=request.query_params,
-        content=body,
-    )
-
-    try:
-        response = await client.send(req, stream=True)
-    except Exception as e:
-        await client.aclose()
-        return StreamingResponse(iter([f"Proxy Error: {e}".encode()]), status_code=502)
-
-    async def stream_and_close():
-        try:
-            async for chunk in response.aiter_raw():
-                yield chunk
-        finally:
-            await response.aclose()
-            await client.aclose()
-
-    res_headers = {
-        k: v for k, v in response.headers.items()
-        if k.lower() not in ("content-length", "transfer-encoding", "connection")
-    }
-
-    return StreamingResponse(
-        stream_and_close(),
-        status_code=response.status_code,
-        headers=res_headers
-    )
-
-
-if __name__ == "__main__":
-    start_llama_server()
-    app.launch(show_error=True)