Update app.py
Browse files
app.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
| 1 |
import os
|
|
|
|
| 2 |
import gradio as gr
|
| 3 |
from llama_cpp import Llama
|
| 4 |
from huggingface_hub import hf_hub_download
|
|
@@ -12,15 +13,23 @@ MODEL_CACHE = "/tmp/qwen_coder.gguf"
|
|
| 12 |
print("Checking model cache...")
|
| 13 |
if not os.path.exists(MODEL_CACHE):
|
| 14 |
print("Downloading Q4_K_M model (first run only)...")
|
| 15 |
-
|
| 16 |
repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
|
| 17 |
filename="qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", # ~2x faster than Q8
|
| 18 |
)
|
| 19 |
-
os.rename(
|
|
|
|
|
|
|
|
|
|
| 20 |
print("Model cached.")
|
| 21 |
else:
|
| 22 |
print("Model found in cache, skipping download.")
|
| 23 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 24 |
print("Loading model into memory...")
|
| 25 |
llm = Llama(
|
| 26 |
model_path=MODEL_CACHE,
|
|
|
|
| 1 |
import os
|
| 2 |
+
import shutil
|
| 3 |
import gradio as gr
|
| 4 |
from llama_cpp import Llama
|
| 5 |
from huggingface_hub import hf_hub_download
|
|
|
|
| 13 |
print("Checking model cache...")
|
| 14 |
if not os.path.exists(MODEL_CACHE):
|
| 15 |
print("Downloading Q4_K_M model (first run only)...")
|
| 16 |
+
downloaded_path = hf_hub_download(
|
| 17 |
repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
|
| 18 |
filename="qwen2.5-coder-1.5b-instruct-q4_k_m.gguf", # ~2x faster than Q8
|
| 19 |
)
|
| 20 |
+
# os.rename() fails across filesystems (e.g. HF cache → /tmp are different partitions)
|
| 21 |
+
# shutil.copy2 works reliably across any filesystems
|
| 22 |
+
print(f"Copying model to cache: {downloaded_path} → {MODEL_CACHE}")
|
| 23 |
+
shutil.copy2(downloaded_path, MODEL_CACHE)
|
| 24 |
print("Model cached.")
|
| 25 |
else:
|
| 26 |
print("Model found in cache, skipping download.")
|
| 27 |
|
| 28 |
+
# Verify the file actually exists and has size before loading
|
| 29 |
+
if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) == 0:
|
| 30 |
+
raise RuntimeError(f"Model cache file missing or empty at {MODEL_CACHE}")
|
| 31 |
+
|
| 32 |
+
print(f"Model file size: {os.path.getsize(MODEL_CACHE) / 1e9:.2f} GB")
|
| 33 |
print("Loading model into memory...")
|
| 34 |
llm = Llama(
|
| 35 |
model_path=MODEL_CACHE,
|