Update app.py

app.py

@@ -32,10 +32,13 @@ def build_latest_llama_server():
     print("Building latest llama.cpp from master source...")
 
     if not os.path.exists("llama.cpp"):
-        subprocess.run(["git", "clone", "https://github.com/ggml-org/llama.cpp.git"], check=True)
+        subprocess.run(["git", "clone", "https://github.com/unslothai/llama.cpp.git"], check=True)
     else:
         subprocess.run(["git", "pull"], cwd="llama.cpp", check=True)
 
+    subprocess.run(["git", "fetch", "origin", "pull/144/head:mtp"], cwd="llama.cpp", check=True)
+    subprocess.run(["git", "checkout", "mtp"], cwd="llama.cpp", check=True)
+
     env = os.environ.copy()
     if "/usr/local/cuda/bin" not in env.get("PATH", ""):
         env["PATH"] = f"/usr/local/cuda/bin:{env.get('PATH', '')}"
@@ -74,31 +77,30 @@ def start_llama_server():
 
     from huggingface_hub import hf_hub_download
 
-    model_path = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
-    )
-
-    model_path2 = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00002-of-00004.gguf",
-    )
-
-    model_path3 = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00003-of-00004.gguf",
-    )
-    model_path4 = hf_hub_download(
-        repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-        filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00004-of-00004.gguf",
+    #model_path = hf_hub_download(
+    #    repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
+    #    filename="UD-Q4_K_XL/Qwen3.8-Flash-Next-UD-Q4_K_XL-00001-of-00004.gguf",
+    #)
+    model_path = None
+    for i in range(1,34):
+        mp = hf_hub_download(
+            repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
+            filename="Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64/Qwen3.8-Flash-Next-AD-4.27bpw-Q4_K_M-M64-{}-of-00033.gguf".format(str(i).zfill(5))
+        )
+        if not model_path:
+            model_path = mp
+
+    mmproj_path = hf_hub_download(
+        repo_id="AtomicChat/Qwen3.8-Flash-Next-GGUF",
+        filename="mmproj-Qwen3.8-Flash-Next-F16.gguf",
     )
-    
 
     #draft_path = hf_hub_download(
     #    repo_id="unsloth/Qwen3.8-Flash-Next-GGUF",
-    #    filename="MTP/mtp-Qwen3.8-Flash-Next-Q4_0.gguf",
+    #    filename="MTP/mtp-Qwen3.8-Flash-Next-Q4_K_M.gguf",
     #)
 
+
     binary_dir = os.path.dirname(os.path.abspath(binary_path))
     env = os.environ.copy()
     env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
@@ -106,15 +108,15 @@ def start_llama_server():
     cmd = [
         binary_path,
         "-m", model_path,
+        "-mm", mmproj_path,
+        #"-md", draft_path,
         "--port", "8000",
         "--host", "127.0.0.1",
         "-t", "16",
         "-tb", "16",
         "-fa", "on",
         "--parallel", "1",
-        "--load-mode", "mmap",
-        #"--spec-type", "draft-mtp",
-        #"--spec-draft-n-max", "4",
+        "--load-mode", "dio",
         "--cache-type-k", "q4_0",
         "--cache-type-v", "q4_0",
         "--jinja",