Create app.py

app.py

@@ -66,7 +66,8 @@ def build_latest_llama_server():
         "-DCMAKE_BUILD_TYPE=Release",
         "-DGGML_NATIVE=ON",
         "-DGGML_OPENMP=ON",
-        "-DGGML_AVX512=OFF",
+        "-DGGML_AVX512=ON",
+        "-DGGML_AMX=OFF",
     ]
 
     subprocess.run(cmake_cmd, cwd="llama.cpp", env=env, check=True)
@@ -120,14 +121,16 @@ def start_llama_server():
     env = os.environ.copy()
     env["LD_LIBRARY_PATH"] = f"{binary_dir}:{env.get('LD_LIBRARY_PATH', '')}"
 
+    n_threads = str(os.cpu_count() or 2)
+
     cmd = [
         binary_path,
         "-m", model_path,
         "-mm", mmproj_path,
         "--port", "8000",
         "--host", "127.0.0.1",
-        "-t", "16",
-        "-tb", "16",
+        "-t", n_threads,
+        "-tb", n_threads,
         "-fa", "on",
         "--parallel", "1",
         "--load-mode", "dio",
@@ -145,10 +148,41 @@ def start_llama_server():
 
     process = subprocess.Popen(cmd, env=env)
     _llama_server_started = True
-    time.sleep(5)
     return process
 
 
+def wait_for_llama_server(process, poll_interval: float = 3.0, log_every: float = 30.0):
+    """Block until llama-server answers on /health, instead of guessing with a
+    fixed sleep. This is what the old code was missing: app.launch() used to
+    run ~5s after starting llama-server no matter what, so Gradio started
+    accepting traffic (and the proxy started forwarding to it) long before a
+    multi-GB / multi-shard model had actually finished loading on CPU - which
+    is exactly what produced "Proxy Error: All connection attempts failed".
+    There is no hard timeout here on purpose: a large CPU-only model can
+    legitimately take several minutes to mmap and load."""
+    print("Waiting for llama-server to finish loading the model before accepting traffic...")
+    waited = 0.0
+    last_log = 0.0
+    while True:
+        if process is not None and process.poll() is not None:
+            raise RuntimeError(
+                f"llama-server exited unexpectedly (return code {process.returncode}) "
+                "while loading the model - scroll up in the Container logs for the real error."
+            )
+        try:
+            r = httpx.get("http://127.0.0.1:8000/health", timeout=5)
+            if r.status_code < 400:
+                print(f"llama-server is ready after ~{waited:.0f}s.")
+                return
+        except httpx.HTTPError:
+            pass
+        time.sleep(poll_interval)
+        waited += poll_interval
+        if waited - last_log >= log_every:
+            print(f"...still loading ({waited:.0f}s elapsed)")
+            last_log = waited
+
+
 @app.middleware("http")
 async def proxy_middleware(request: Request, call_next):
     path = request.url.path
@@ -199,5 +233,6 @@ async def proxy_middleware(request: Request, call_next):
 
 
 if __name__ == "__main__":
-    start_llama_server()
+    llama_process = start_llama_server()
+    wait_for_llama_server(llama_process)
     app.launch(show_error=True)