Spaces:
Paused
Paused
Download modules/llm/qwen.py from SamilG2/BACKEND-CheckScan: direct link, hf CLI and curl.
- Browser
- Download file 3.62 kB
-
https://huggingface.co/spaces/SamilG2/BACKEND-CheckScan/resolve/main/modules/llm/qwen.py
- Command line
-
hf download hf://spaces/SamilG2/BACKEND-CheckScan/modules/llm/qwen.py
-
curl -L -o qwen.py https://huggingface.co/spaces/SamilG2/BACKEND-CheckScan/resolve/main/modules/llm/qwen.py
3.62 kB
| """Singleton lazy para Qwen2.5-3B-Instruct vía llama-cpp-python (GGUF Q4_K_M). | |
| El modelo se carga en la primera llamada y se reutiliza entre requests. | |
| Requiere el archivo GGUF en la ruta configurada en config.QWEN_MODEL_PATH. | |
| Descarga: | |
| hf download Qwen/Qwen2.5-3B-Instruct-GGUF qwen2.5-3b-instruct-q4_k_m.gguf \\ | |
| --local-dir data/models/ | |
| """ | |
| import os | |
| import threading | |
| from typing import TYPE_CHECKING | |
| import torch | |
| from modules.llm.prompt_builder import SYSTEM | |
| if TYPE_CHECKING: # solo para los type checkers | |
| from llama_cpp import Llama | |
| _model: "Llama | None" = None | |
| _load_lock = threading.Lock() | |
| _infer_lock = threading.Lock() # llama-cpp no es thread-safe | |
| def _resolve_model_path(local_path: str) -> str: | |
| """Devuelve la ruta local del GGUF. Si no existe (deploy: el modelo no está | |
| en el repo por peso), lo descarga de HF Hub —mismo patrón que TrOCR— y cachea. | |
| En local, si el archivo ya está, lo usa tal cual (sin red).""" | |
| if os.path.isfile(local_path): | |
| return local_path | |
| from config import QWEN_COACH_REPO, QWEN_COACH_FILENAME | |
| from huggingface_hub import hf_hub_download | |
| print(f"[Qwen] '{local_path}' no está localmente. Descargando " | |
| f"'{QWEN_COACH_FILENAME}' de '{QWEN_COACH_REPO}' (HF Hub)...") | |
| path = hf_hub_download( | |
| repo_id=QWEN_COACH_REPO, | |
| filename=QWEN_COACH_FILENAME, | |
| token=os.getenv("HF_TOKEN"), # None para repos públicos | |
| ) | |
| print(f"[Qwen] Modelo descargado en {path}") | |
| return path | |
| def _load() -> None: | |
| global _model | |
| if _model is not None: | |
| return | |
| with _load_lock: | |
| if _model is not None: # re-check inside lock | |
| return | |
| # llama_cpp se importa aquí y no arriba: así el backend arranca sin la | |
| # dependencia cuando COACH_MODEL=deterministic (experimento A). | |
| from llama_cpp import Llama | |
| from config import QWEN_MODEL_PATH | |
| model_path = _resolve_model_path(QWEN_MODEL_PATH) | |
| cuda = torch.cuda.is_available() | |
| n_gpu = int(os.getenv("QWEN_GPU_LAYERS", "-1" if cuda else "0")) | |
| print(f"[Qwen] Cargando modelo: {model_path} ({'GPU' if cuda else 'CPU'}, gpu_layers={n_gpu})") | |
| try: | |
| _model = Llama( | |
| model_path=model_path, | |
| n_ctx=2048, | |
| n_threads=int(os.getenv("QWEN_NUM_THREADS", "6")), | |
| n_gpu_layers=n_gpu, | |
| verbose=False, | |
| ) | |
| except Exception as e: | |
| print(f"[Qwen] ERROR al construir Llama: {type(e).__name__}: {e}") | |
| raise | |
| print("[Qwen] Modelo listo.") | |
| def generate(prompt: str, system: str = SYSTEM, max_tokens: int = 150) -> str: | |
| """Genera texto con el modelo. `system` y `max_tokens` permiten reutilizar el | |
| mismo modelo para el comentario corto y para el expandido (más largo).""" | |
| _load() | |
| with _infer_lock: | |
| return _generate_locked(prompt, system, max_tokens) | |
| def _generate_locked(prompt: str, system: str, max_tokens: int) -> str: | |
| import time | |
| print(f"[Qwen] >>> generando (prompt {len(prompt)} chars)...", flush=True) | |
| t0 = time.time() | |
| out = _model.create_chat_completion( | |
| messages=[ | |
| {"role": "system", "content": system}, | |
| {"role": "user", "content": prompt}, | |
| ], | |
| max_tokens=max_tokens, | |
| temperature=0.35, | |
| stop=["<|im_end|>", "\n\n\n"], | |
| ) | |
| dt = time.time() - t0 | |
| text = out["choices"][0]["message"]["content"].strip() | |
| print(f"[Qwen] <<< {dt:.1f}s | {len(text)} chars: {text[:80]!r}", flush=True) | |
| return text | |