Update app.py

app.py

@@ -1,11 +1,8 @@
 from __future__ import annotations
-import json
+import glob
 import os
 import subprocess
-import tarfile
 import time
-import urllib.request
-import zipfile
 
 from fastapi import Request
 from fastapi.responses import StreamingResponse
@@ -24,6 +21,7 @@ except ImportError:
 def whygpu():
     pass
 
+
 def build_latest_llama_server():
     binary_path = os.path.abspath("llama.cpp/build/bin/llama-server")
     if os.path.exists(binary_path):
@@ -36,8 +34,13 @@ def build_latest_llama_server():
     else:
         subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
 
-    subprocess.run(["git", "fetch", "origin", "pull/144/head:mtp"], cwd="llama.cpp", check=True)
-    subprocess.run(["git", "checkout", "mtp"], cwd="llama.cpp", check=True)
+    # NOTE: no longer fetching/checking out the "mtp" branch (pull/144) here.
+    # That branch was specific to Qwen3.8-Flash-Next's MTP support and has
+    # nothing to do with GLM-5.3-Flash's architecture (glm5next). Keeping it
+    # would make llama.cpp fail to load this model. unslothai/llama.cpp's
+    # default branch already carries GLM-5.3-Flash support (PR ggml-org#27754
+    # is merged into their regular mix builds), so a plain clone/pull is
+    # enough. Everything else below is unchanged from your original.
 
     env = os.environ.copy()
     if "/usr/local/cuda/bin" not in env.get("PATH", ""):
@@ -58,7 +61,7 @@ def build_latest_llama_server():
         ["cmake", "--build", "build", "--config", "Release", "-j", str(os.cpu_count() or 4), "--target", "llama-server"],
         cwd="llama.cpp",
         env=env,
-        check=True
+        check=True,
     )
 
     if not os.path.exists(binary_path):
@@ -66,40 +69,69 @@ def build_latest_llama_server():
 
     return binary_path
 
-def apathy_exe():
-    model_path = hf_hub_download(
-        repo_id="Qwen/Qwen3.8-Flash-Next",
-        filename="q.gguf",
+
+# ---------------------------------------------------------------------------
+# Model: GLM-5.3-Flash (Z.ai), GGUF build by unsloth.
+#
+# Why this model: as of the strongest *practically self-hostable* open-weight
+# agent model with an MIT license is GLM-5.3-Flash (320B total / 18B active,
+# natively multimodal, hybrid sparse+linear attention). It leads open-weight
+# results on agentic/coding benchmarks (Terminal-Bench 2.1, DeepSWE,
+# Toolathlon-Verified) and is reported to approach Claude Opus 4.8 on
+# coding/agentic tasks. Moonshot's Kimi K3 (2.8T) currently benchmarks a
+# notch higher, but needs ~600GB+ of RAM/VRAM even at its smallest quant —
+# not realistic for a single-GPU Space, so it isn't used here.
+#
+# QUANT: pick the smallest quant that fits your box. Sizes (unsloth Dynamic
+# GGUF): UD-IQ1_S ≈93GB, UD-IQ1_M ≈98GB, UD-IQ2_XXS ≈102GB, UD-Q2_K_XL ≈109GB,
+# UD-Q3_K_XL ≈148GB, UD-Q4_K_XL ≈200GB. Even the smallest quant will not fit
+# on a free/ZeroGPU Space's default disk — you need a machine with enough
+# disk + RAM (llama.cpp can stream/offload experts to CPU RAM and disk, it
+# doesn't all need to be in VRAM, but it does need to fit somewhere).
+# ---------------------------------------------------------------------------
+MODEL_REPO = "unsloth/GLM-5.3-Flash-GGUF"
+MODEL_QUANT = "UD-IQ1_S"  # smallest available quant; raise if you have the disk/RAM
+
+
+def download_model_shards(repo_id: str, quant_dir: str) -> str:
+    """Downloads every shard of a (possibly multi-part) GGUF quant and
+    returns the path to the first shard, which is what llama-server needs
+    (it auto-discovers the rest of the shards next to it)."""
+    from huggingface_hub import snapshot_download
+
+    local_dir = snapshot_download(
+        repo_id=repo_id,
+        allow_patterns=[f"{quant_dir}/*"],
     )
+    shard_dir = os.path.join(local_dir, quant_dir)
+    shards = sorted(glob.glob(os.path.join(shard_dir, "*-00001-of-*.gguf")))
+    if not shards:
+        # not sharded — just one file in the folder
+        shards = sorted(glob.glob(os.path.join(shard_dir, "*.gguf")))
+    if not shards:
+        raise RuntimeError(f"No GGUF files found under {shard_dir}")
+    return shards[0]
+
+
+def download_mmproj(repo_id: str) -> str:
+    """GLM-5.3-Flash is natively multimodal; grab the F16 mmproj file so
+    image input works with -mm, same as your original -mm usage."""
+    from huggingface_hub import hf_hub_download
 
-def start_llama_server():
-    binary_path = build_latest_llama_server()
+    return hf_hub_download(repo_id=repo_id, filename="mmproj-F16.gguf")
 
-    from huggingface_hub import hf_hub_download
 
-    model_path = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="BF16/Qwen3.8-Flash-Next-BF16-00008-of-00008.gguf",
-    )
-    model_path = None
-    for i in range(1,34):
-       # mp = hf_hub_download(
-        #    repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
-       #     filename="Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64/Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64-{}-of-00033.gguf".format(str(i).zfill(5))
-     #   )
-        if not model_path:
-          #  model_path = mp
-
-  #  mmproj_path = hf_hub_download(
-     #   repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
-     #   filename="mmproj-Qwen3.8-Flash-Next-F16.gguf",
-   # )
-
-    draft_path = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="MTP/mtp-Qwen3.8-Flash-Next-Q4_K_M.gguf",
-    )
+def start_llama_server():
+    binary_path = build_latest_llama_server()
 
+    model_path = download_model_shards(MODEL_REPO, MODEL_QUANT)
+    mmproj_path = download_mmproj(MODEL_REPO)
+    # No separate draft_path/-md here: unlike Qwen3.8-Flash-Next, GLM-5.3-Flash
+    # doesn't ship (or need) a separate MTP draft GGUF - MTP is built into the
+    # main file via the llama.cpp PR mentioned above. There is no matching
+    # file to download from the repo, so the old draft_path download and the
+    # (already-commented-out) "-md" flag are simply gone rather than pointing
+    # at a file that doesn't exist.
 
     binary_dir = os.path.dirname(os.path.abspath(binary_path))
     env = os.environ.copy()
@@ -109,7 +141,6 @@ def start_llama_server():
         binary_path,
         "-m", model_path,
         "-mm", mmproj_path,
-        #"-md", draft_path,
         "--port", "8000",
         "--host", "127.0.0.1",
         "-t", "16",
@@ -132,6 +163,7 @@ def start_llama_server():
     time.sleep(5)
     return process
 
+
 @app.middleware("http")
 async def proxy_middleware(request: Request, call_next):
     path = request.url.path
@@ -139,9 +171,9 @@ async def proxy_middleware(request: Request, call_next):
         return await call_next(request)
 
     url = f"http://127.0.0.1:8000{path}"
-    
+
     headers = {
-        k: v for k, v in request.headers.items() 
+        k: v for k, v in request.headers.items()
         if k.lower() not in ("host", "content-length", "accept-encoding")
     }
     body = await request.body()
@@ -177,9 +209,10 @@ async def proxy_middleware(request: Request, call_next):
     return StreamingResponse(
         stream_and_close(),
         status_code=response.status_code,
-        headers=res_headers
+        headers=res_headers,
     )
 
+
 if __name__ == "__main__":
     start_llama_server()
     app.launch(show_error=True)