cazyundee commited on
Commit
f3c6ec5
·
verified ·
1 Parent(s): 1b0a401

Rebuild llama.cpp with stable CPU backend

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -200,7 +200,7 @@ _BENCH_MODELS = [
200
  {"name": "Qwen3 8B", "repo": "Qwen/Qwen3-8B-GGUF", "file": "Qwen3-8B-Q4_K_M.gguf", "size_gb": 5.0},
201
  {"name": "Qwen3.8 27B", "repo": "unsloth/Qwen3.8-27B-GGUF", "file": "Qwen3.8-27B-Q4_0.gguf", "size_gb": 16.0},
202
  ]
203
- _BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench")
204
  _BENCH_RESULTS = []
205
  _BENCH_LOCK = threading.Lock()
206
  _BENCH_JOBS = {}
@@ -221,7 +221,7 @@ def _ensure_llama():
221
  bd = os.path.join(_BENCH_CACHE, "build")
222
  os.makedirs(bd, exist_ok=True)
223
  nt = _parse_cpu_count() or 8
224
- subprocess.check_call(["cmake","-S",src,"-B",bd,"-DLLAMA_NATIVE=ON","-DLLAMA_CUDA=OFF","-DGGML_CUDA=OFF","-DBUILD_SHARED_LIBS=OFF"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
225
  subprocess.check_call(["cmake","--build",bd,"-j",str(nt),"--target","llama-cli","llama-bench"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
226
  import glob as _g
227
  cands = [p for p in _g.glob(os.path.join(bd,"**","llama-cli"), recursive=True) if os.path.isfile(p)]
 
200
  {"name": "Qwen3 8B", "repo": "Qwen/Qwen3-8B-GGUF", "file": "Qwen3-8B-Q4_K_M.gguf", "size_gb": 5.0},
201
  {"name": "Qwen3.8 27B", "repo": "unsloth/Qwen3.8-27B-GGUF", "file": "Qwen3.8-27B-Q4_0.gguf", "size_gb": 16.0},
202
  ]
203
+ _BENCH_CACHE = os.path.join(tempfile.gettempdir(), "llm_bench_cpu_v2")
204
  _BENCH_RESULTS = []
205
  _BENCH_LOCK = threading.Lock()
206
  _BENCH_JOBS = {}
 
221
  bd = os.path.join(_BENCH_CACHE, "build")
222
  os.makedirs(bd, exist_ok=True)
223
  nt = _parse_cpu_count() or 8
224
+ subprocess.check_call(["cmake","-S",src,"-B",bd,"-DGGML_NATIVE=OFF","-DGGML_OPENMP=OFF","-DGGML_CUDA=OFF","-DGGML_AVX=ON","-DGGML_AVX2=ON","-DGGML_FMA=ON","-DGGML_AVX512=OFF","-DBUILD_SHARED_LIBS=OFF"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
225
  subprocess.check_call(["cmake","--build",bd,"-j",str(nt),"--target","llama-cli","llama-bench"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
226
  import glob as _g
227
  cands = [p for p in _g.glob(os.path.join(bd,"**","llama-cli"), recursive=True) if os.path.isfile(p)]