Spaces:
Paused
Paused
perf: GPU support for Qwen and TrOCR (T4 / CPU fallback auto)
Browse files- Dockerfile +3 -3
- modules/llm/qwen.py +6 -2
Dockerfile
CHANGED
|
@@ -15,9 +15,9 @@ WORKDIR /app
|
|
| 15 |
COPY requirements.txt .
|
| 16 |
RUN pip install --no-cache-dir --upgrade pip \
|
| 17 |
&& pip install --no-cache-dir numpy==1.26.0 \
|
| 18 |
-
&& pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/
|
| 19 |
-
&&
|
| 20 |
-
|
| 21 |
&& pip install --no-cache-dir -r requirements.txt
|
| 22 |
|
| 23 |
# Pre-descarga del modelo Qwen GGUF en build (arranque r谩pido en runtime)
|
|
|
|
| 15 |
COPY requirements.txt .
|
| 16 |
RUN pip install --no-cache-dir --upgrade pip \
|
| 17 |
&& pip install --no-cache-dir numpy==1.26.0 \
|
| 18 |
+
&& pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cu121 \
|
| 19 |
+
&& pip install --no-cache-dir llama-cpp-python==0.3.2 \
|
| 20 |
+
--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 \
|
| 21 |
&& pip install --no-cache-dir -r requirements.txt
|
| 22 |
|
| 23 |
# Pre-descarga del modelo Qwen GGUF en build (arranque r谩pido en runtime)
|
modules/llm/qwen.py
CHANGED
|
@@ -8,6 +8,7 @@ Descarga:
|
|
| 8 |
--local-dir data/models/
|
| 9 |
"""
|
| 10 |
import os
|
|
|
|
| 11 |
from llama_cpp import Llama
|
| 12 |
from modules.llm.prompt_builder import SYSTEM
|
| 13 |
|
|
@@ -25,12 +26,15 @@ def _load() -> None:
|
|
| 25 |
"Descargalo de huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF "
|
| 26 |
"(archivo qwen2.5-3b-instruct-q4_k_m.gguf) y col贸calo en data/models/."
|
| 27 |
)
|
| 28 |
-
|
|
|
|
|
|
|
| 29 |
try:
|
| 30 |
_model = Llama(
|
| 31 |
model_path=QWEN_MODEL_PATH,
|
| 32 |
n_ctx=2048,
|
| 33 |
-
n_threads=
|
|
|
|
| 34 |
verbose=False,
|
| 35 |
)
|
| 36 |
except Exception as e:
|
|
|
|
| 8 |
--local-dir data/models/
|
| 9 |
"""
|
| 10 |
import os
|
| 11 |
+
import torch
|
| 12 |
from llama_cpp import Llama
|
| 13 |
from modules.llm.prompt_builder import SYSTEM
|
| 14 |
|
|
|
|
| 26 |
"Descargalo de huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF "
|
| 27 |
"(archivo qwen2.5-3b-instruct-q4_k_m.gguf) y col贸calo en data/models/."
|
| 28 |
)
|
| 29 |
+
cuda = torch.cuda.is_available()
|
| 30 |
+
n_gpu = int(os.getenv("QWEN_GPU_LAYERS", "-1" if cuda else "0"))
|
| 31 |
+
print(f"[Qwen] Cargando modelo: {QWEN_MODEL_PATH} ({'GPU' if cuda else 'CPU'}, gpu_layers={n_gpu})")
|
| 32 |
try:
|
| 33 |
_model = Llama(
|
| 34 |
model_path=QWEN_MODEL_PATH,
|
| 35 |
n_ctx=2048,
|
| 36 |
+
n_threads=int(os.getenv("QWEN_NUM_THREADS", "6")),
|
| 37 |
+
n_gpu_layers=n_gpu,
|
| 38 |
verbose=False,
|
| 39 |
)
|
| 40 |
except Exception as e:
|