SamilG2 commited on
Commit
1607935
路
1 Parent(s): b34a06b

perf: GPU support for Qwen and TrOCR (T4 / CPU fallback auto)

Browse files
Files changed (2) hide show
  1. Dockerfile +3 -3
  2. modules/llm/qwen.py +6 -2
Dockerfile CHANGED
@@ -15,9 +15,9 @@ WORKDIR /app
15
  COPY requirements.txt .
16
  RUN pip install --no-cache-dir --upgrade pip \
17
  && pip install --no-cache-dir numpy==1.26.0 \
18
- && pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cpu \
19
- && CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_AVX2=OFF -DGGML_AVX512=OFF -DGGML_FMA=OFF -DGGML_F16C=OFF" \
20
- pip install --no-cache-dir llama-cpp-python==0.3.2 \
21
  && pip install --no-cache-dir -r requirements.txt
22
 
23
  # Pre-descarga del modelo Qwen GGUF en build (arranque r谩pido en runtime)
 
15
  COPY requirements.txt .
16
  RUN pip install --no-cache-dir --upgrade pip \
17
  && pip install --no-cache-dir numpy==1.26.0 \
18
+ && pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/cu121 \
19
+ && pip install --no-cache-dir llama-cpp-python==0.3.2 \
20
+ --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu121 \
21
  && pip install --no-cache-dir -r requirements.txt
22
 
23
  # Pre-descarga del modelo Qwen GGUF en build (arranque r谩pido en runtime)
modules/llm/qwen.py CHANGED
@@ -8,6 +8,7 @@ Descarga:
8
  --local-dir data/models/
9
  """
10
  import os
 
11
  from llama_cpp import Llama
12
  from modules.llm.prompt_builder import SYSTEM
13
 
@@ -25,12 +26,15 @@ def _load() -> None:
25
  "Descargalo de huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF "
26
  "(archivo qwen2.5-3b-instruct-q4_k_m.gguf) y col贸calo en data/models/."
27
  )
28
- print(f"[Qwen] Cargando modelo: {QWEN_MODEL_PATH}")
 
 
29
  try:
30
  _model = Llama(
31
  model_path=QWEN_MODEL_PATH,
32
  n_ctx=2048,
33
- n_threads=4,
 
34
  verbose=False,
35
  )
36
  except Exception as e:
 
8
  --local-dir data/models/
9
  """
10
  import os
11
+ import torch
12
  from llama_cpp import Llama
13
  from modules.llm.prompt_builder import SYSTEM
14
 
 
26
  "Descargalo de huggingface.co/Qwen/Qwen2.5-3B-Instruct-GGUF "
27
  "(archivo qwen2.5-3b-instruct-q4_k_m.gguf) y col贸calo en data/models/."
28
  )
29
+ cuda = torch.cuda.is_available()
30
+ n_gpu = int(os.getenv("QWEN_GPU_LAYERS", "-1" if cuda else "0"))
31
+ print(f"[Qwen] Cargando modelo: {QWEN_MODEL_PATH} ({'GPU' if cuda else 'CPU'}, gpu_layers={n_gpu})")
32
  try:
33
  _model = Llama(
34
  model_path=QWEN_MODEL_PATH,
35
  n_ctx=2048,
36
+ n_threads=int(os.getenv("QWEN_NUM_THREADS", "6")),
37
+ n_gpu_layers=n_gpu,
38
  verbose=False,
39
  )
40
  except Exception as e: