Upload app_deepseek.py

app_deepseek.py

@@ -401,7 +401,10 @@ GGUF_REPO = os.environ.get("GIJL_GGUF_REPO", "unsloth/DeepSeek-V4-Flash-0731-GGU
 # GIJL_GGUF_QUANT to override explicitly.
 GGUF_QUANT = os.environ.get("GIJL_GGUF_QUANT", "UD-IQ3_XXS")
 LLAMA_CTX = int(os.environ.get("GIJL_CTX", "12288"))            # tokens; KV cache is ~2-4GB at this size
-LLAMA_LOAD_TIMEOUT_S = 1800
+LLAMA_LOAD_TIMEOUT_S = 3600   # was 1800s; a ~100GB GGUF loading on CPU-only (mmap + page faults, no GPU to offload
+                               # onto) can legitimately take well past 30 minutes depending on storage throughput --
+                               # this is not an error state (see the "[llama] loading model ... Ns" progress log),
+                               # just slow I/O, so the budget needs real headroom instead of racing it.
 LLAMA_BIN_DIR = os.path.join(WORK_DIR, "bin")                   # CUDA build cache (synced to the hub, see CACHE_LLAMA_BINARY)
 LLAMA_CPU_DIR = os.path.join(WORK_DIR, "llama_cpu")             # prebuilt CPU release (not synced: quick to re-download)
 LLAMA_BUILD_ROOT = os.environ.get("GIJL_LLAMA_BUILD_DIR", "/tmp/gijl_llama_build")
@@ -1424,20 +1427,37 @@ def build_llama_cuda(archs: list):
 
 def download_llama_cpu_release():
     """Prebuilt CPU llama-server from GitHub releases. The 'latest' release only carries nightly-tag.txt, which
-    names the tag holding the real binaries (llama-<tag>-bin-ubuntu-x64.tar.gz)."""
+    names the tag holding the real binaries (llama-<tag>-bin-ubuntu-x64.tar.gz).
+    GIJL_LLAMA_CPU_TAG pins a specific release tag (e.g. "b10800") instead of always grabbing whatever is newest.
+    This exists because of a confirmed, UNFIXED upstream llama.cpp bug: certain large/MoE GGUF models crash
+    CPU-only inference with GGML_ASSERT(*cur_backend_id != -1) in ggml_backend_sched_split_graph during context
+    creation -- reproduced against the build this pipeline downloads (b11429) and reported against several other
+    recent builds too (ggml-org/llama.cpp issues #18099, #19397), closed upstream as "not planned" with no fix and
+    no confirmed workaround other than attaching a GPU. -fit off does NOT fix it (tested in both issues and in
+    this pipeline's own run logs). If you hit that exact crash, there is nothing further this Python script can
+    patch around -- it is a C++ scheduler bug, not a flag the server exposes. Two real options: (1) run with a GPU
+    accelerator (this pipeline already warns for that at startup), or (2) set GIJL_LLAMA_CPU_TAG to an older
+    release tag that predates whatever regression introduced this, and see whether that specific build is
+    unaffected for your model/quant -- there is no single tag known in advance to be safe, so this is trial and
+    error, not a guaranteed fix."""
     import requests
     arch = "arm64" if platform.machine().lower() in ("aarch64", "arm64") else "x64"
     base = "https://github.com/ggml-org/llama.cpp/releases"
     hdr = {"User-Agent": _CRAWLER_UA}
-    tags = []
-    try:
-        latest = requests.get(f"{base}/latest", timeout=30, headers=hdr).url.rstrip("/").rsplit("/", 1)[-1]
-        nt = requests.get(f"{base}/download/{latest}/nightly-tag.txt", timeout=30, headers=hdr)
-        if nt.status_code == 200 and nt.text.strip():
-            tags.append(nt.text.strip().split()[0])
-        tags.append(latest)
-    except requests.RequestException as e:
-        raise RuntimeError(f"cannot reach GitHub releases: {_redact(str(e))[:200]}")
+    pinned = os.environ.get("GIJL_LLAMA_CPU_TAG", "").strip()
+    if pinned:
+        print(f"[llama] GIJL_LLAMA_CPU_TAG set -> using pinned CPU build {pinned} instead of latest", flush=True)
+        tags = [pinned]
+    else:
+        tags = []
+        try:
+            latest = requests.get(f"{base}/latest", timeout=30, headers=hdr).url.rstrip("/").rsplit("/", 1)[-1]
+            nt = requests.get(f"{base}/download/{latest}/nightly-tag.txt", timeout=30, headers=hdr)
+            if nt.status_code == 200 and nt.text.strip():
+                tags.append(nt.text.strip().split()[0])
+            tags.append(latest)
+        except requests.RequestException as e:
+            raise RuntimeError(f"cannot reach GitHub releases: {_redact(str(e))[:200]}")
     last_err = "no candidate tag"
     for tag in tags:
         dest = os.path.join(LLAMA_CPU_DIR, tag)
@@ -1591,7 +1611,21 @@ class LlamaServer:
         t0, last = time.time(), time.time()
         while time.time() - t0 < LLAMA_LOAD_TIMEOUT_S:
             if self.proc.poll() is not None:
-                raise RuntimeError(f"llama-server exited during startup (code {self.proc.returncode}). Log tail:\n{self.tail(40)}")
+                tail = self.tail(40)
+                if self.kind == "cpu" and "cur_backend_id != -1" in tail:
+                    # Confirmed unfixed upstream llama.cpp bug, not something this script's flags control -- see the
+                    # long comment on download_llama_cpu_release(). Spell this out instead of just dumping the
+                    # C++ backtrace, since "-fit off" (tried automatically above) is already known NOT to fix it.
+                    raise RuntimeError(
+                        "llama-server crashed during model load with GGML_ASSERT(*cur_backend_id != -1) failed "
+                        "(ggml_backend_sched_split_graph). This is a confirmed, UNFIXED upstream llama.cpp bug on "
+                        "CPU-only inference of some large/MoE GGUF models (ggml-org/llama.cpp issues #18099, "
+                        "#19397 -- closed as 'not planned', no fix released). --fit off does not fix it. There is "
+                        "no flag this script can add to work around it. Real options: (1) run on a GPU accelerator "
+                        "instead of CPU, or (2) set GIJL_LLAMA_CPU_TAG to try a different/older llama-server CPU "
+                        f"release build (current: {os.path.basename(os.path.dirname(self.binary)) or '?'}), with no "
+                        f"guarantee any particular tag is unaffected. Log tail:\n{tail}")
+                raise RuntimeError(f"llama-server exited during startup (code {self.proc.returncode}). Log tail:\n{tail}")
             try:
                 r = requests.get(self.base_url + "/health", timeout=5)
                 if r.status_code == 200 and r.json().get("status") == "ok":