Delete app.py

app.py

@@ -1,218 +0,0 @@
-from __future__ import annotations
-import glob
-import os
-import subprocess
-import time
-
-from fastapi import Request
-from fastapi.responses import StreamingResponse
-from gradio import Server
-import httpx
-
-app = Server()
-
-try:
-    import spaces
-    HAS_SPACES = True
-except ImportError:
-    HAS_SPACES = False
-
-@spaces.GPU if HAS_SPACES else lambda f: f
-def whygpu():
-    pass
-
-
-def build_latest_llama_server():
-    binary_path = os.path.abspath("llama.cpp/build/bin/llama-server")
-    if os.path.exists(binary_path):
-        return binary_path
-
-    print("Building latest llama.cpp from master source...")
-
-    if not os.path.exists("llama.cpp"):
-        subprocess.run(["git", "clone", "https://github.com/unslothai/llama.cpp.git"], check=True)
-    else:
-        subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
-
-    # NOTE: no longer fetching/checking out the "mtp" branch (pull/144) here.
-    # That branch was specific to Qwen3.8-Flash-Next's MTP support and has
-    # nothing to do with GLM-5.3-Flash's architecture (glm5next). Keeping it
-    # would make llama.cpp fail to load this model. unslothai/llama.cpp's
-    # default branch already carries GLM-5.3-Flash support (PR ggml-org#27754
-    # is merged into their regular mix builds), so a plain clone/pull is
-    # enough. Everything else below is unchanged from your original.
-
-    env = os.environ.copy()
-    if "/usr/local/cuda/bin" not in env.get("PATH", ""):
-        env["PATH"] = f"/usr/local/cuda/bin:{env.get('PATH', '')}"
-
-    cmake_cmd = [
-        "cmake",
-        "-B",
-        "build",
-        "-DCMAKE_BUILD_TYPE=Release",
-        "-DGGML_NATIVE=ON",
-        "-DGGML_OPENMP=ON",
-        "-DGGML_AVX512=ON",
-    ]
-
-    subprocess.run(cmake_cmd, cwd="llama.cpp", env=env, check=True)
-    subprocess.run(
-        ["cmake", "--build", "build", "--config", "Release", "-j", str(os.cpu_count() or 4), "--target", "llama-server"],
-        cwd="llama.cpp",
-        env=env,
-        check=True,
-    )
-
-    if not os.path.exists(binary_path):
-        raise RuntimeError("Failed to build llama-server binary!")
-
-    return binary_path
-
-
-# ---------------------------------------------------------------------------
-# Model: GLM-5.3-Flash (Z.ai), GGUF build by unsloth.
-#
-# Why this model: as of the strongest *practically self-hostable* open-weight
-# agent model with an MIT license is GLM-5.3-Flash (320B total / 18B active,
-# natively multimodal, hybrid sparse+linear attention). It leads open-weight
-# results on agentic/coding benchmarks (Terminal-Bench 2.1, DeepSWE,
-# Toolathlon-Verified) and is reported to approach Claude Opus 4.8 on
-# coding/agentic tasks. Moonshot's Kimi K3 (2.8T) currently benchmarks a
-# notch higher, but needs ~600GB+ of RAM/VRAM even at its smallest quant —
-# not realistic for a single-GPU Space, so it isn't used here.
-#
-# QUANT: pick the smallest quant that fits your box. Sizes (unsloth Dynamic
-# GGUF): UD-IQ1_S ≈93GB, UD-IQ1_M ≈98GB, UD-IQ2_XXS ≈102GB, UD-Q2_K_XL ≈109GB,
-# UD-Q3_K_XL ≈148GB, UD-Q4_K_XL ≈200GB. Even the smallest quant will not fit
-# on a free/ZeroGPU Space's default disk — you need a machine with enough
-# disk + RAM (llama.cpp can stream/offload experts to CPU RAM and disk, it
-# doesn't all need to be in VRAM, but it does need to fit somewhere).
-# ---------------------------------------------------------------------------
-MODEL_REPO = "unsloth/GLM-5.3-Flash-GGUF"
-MODEL_QUANT = "UD-IQ1_S"  # smallest available quant; raise if you have the disk/RAM
-
-
-def download_model_shards(repo_id: str, quant_dir: str) -> str:
-    """Downloads every shard of a (possibly multi-part) GGUF quant and
-    returns the path to the first shard, which is what llama-server needs
-    (it auto-discovers the rest of the shards next to it)."""
-    from huggingface_hub import snapshot_download
-
-    local_dir = snapshot_download(
-        repo_id=repo_id,
-        allow_patterns=[f"{quant_dir}/*"],
-    )
-    shard_dir = os.path.join(local_dir, quant_dir)
-    shards = sorted(glob.glob(os.path.join(shard_dir, "*-00001-of-*.gguf")))
-    if not shards:
-        # not sharded — just one file in the folder
-        shards = sorted(glob.glob(os.path.join(shard_dir, "*.gguf")))
-    if not shards:
-        raise RuntimeError(f"No GGUF files found under {shard_dir}")
-    return shards[0]
-
-
-def download_mmproj(repo_id: str) -> str:
-    """GLM-5.3-Flash is natively multimodal; grab the F16 mmproj file so
-    image input works with -mm, same as your original -mm usage."""
-    from huggingface_hub import hf_hub_download
-
-    return hf_hub_download(repo_id=repo_id, filename="mmproj-F16.gguf")
-
-
-def start_llama_server():
-    binary_path = build_latest_llama_server()
-
-    model_path = download_model_shards(MODEL_REPO, MODEL_QUANT)
-    mmproj_path = download_mmproj(MODEL_REPO)
-    # No separate draft_path/-md here: unlike Qwen3.8-Flash-Next, GLM-5.3-Flash
-    # doesn't ship (or need) a separate MTP draft GGUF - MTP is built into the
-    # main file via the llama.cpp PR mentioned above. There is no matching
-    # file to download from the repo, so the old draft_path download and the
-    # (already-commented-out) "-md" flag are simply gone rather than pointing
-    # at a file that doesn't exist.
-
-    binary_dir = os.path.dirname(os.path.abspath(binary_path))
-    env = os.environ.copy()
-    env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
-
-    cmd = [
-        binary_path,
-        "-m", model_path,
-        "-mm", mmproj_path,
-        "--port", "8000",
-        "--host", "127.0.0.1",
-        "-t", "16",
-        "-tb", "16",
-        "-fa", "on",
-        "--parallel", "1",
-        "--load-mode", "dio",
-      #  "--cache-type-k", "q4_0",
-       # "--cache-type-v", "q4_0",
-        "--jinja",
-        "--temp", "1.0",
-        "--top-p", "0.95",
-        "--top-k", "20",
-        "--min-p", "0.0",
-        "--presence-penalty", "0.0",
-        "--repeat-penalty", "1.0",
-    ]
-
-    process = subprocess.Popen(cmd, env=env)
-    time.sleep(5)
-    return process
-
-
-@app.middleware("http")
-async def proxy_middleware(request: Request, call_next):
-    path = request.url.path
-    if path.startswith("/gradio_api"):
-        return await call_next(request)
-
-    url = f"http://127.0.0.1:8000{path}"
-
-    headers = {
-        k: v for k, v in request.headers.items()
-        if k.lower() not in ("host", "content-length", "accept-encoding")
-    }
-    body = await request.body()
-
-    client = httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=120.0))
-    req = client.build_request(
-        method=request.method,
-        url=url,
-        headers=headers,
-        params=request.query_params,
-        content=body,
-    )
-
-    try:
-        response = await client.send(req, stream=True)
-    except Exception as e:
-        await client.aclose()
-        return StreamingResponse(iter([f"Proxy Error: {e}".encode()]), status_code=502)
-
-    async def stream_and_close():
-        try:
-            async for chunk in response.aiter_raw():
-                yield chunk
-        finally:
-            await response.aclose()
-            await client.aclose()
-
-    res_headers = {
-        k: v for k, v in response.headers.items()
-        if k.lower() not in ("content-length", "transfer-encoding", "connection")
-    }
-
-    return StreamingResponse(
-        stream_and_close(),
-        status_code=response.status_code,
-        headers=res_headers,
-    )
-
-
-if __name__ == "__main__":
-    start_llama_server()
-    app.launch(show_error=True)