Spaces:
Paused
Paused
Download neural_forge/models.py from drrobot9/neural-forge: direct link, hf CLI and curl.
- Browser
- Download file 24.9 kB
-
https://huggingface.co/spaces/drrobot9/neural-forge/resolve/acf94deef34555bdff29db9012f442cbd225d8e8/neural_forge/models.py
- Command line
-
hf download hf://spaces/drrobot9/neural-forge@acf94deef34555bdff29db9012f442cbd225d8e8/neural_forge/models.py
-
curl -L -o models.py https://huggingface.co/spaces/drrobot9/neural-forge/resolve/acf94deef34555bdff29db9012f442cbd225d8e8/neural_forge/models.py
24.9 kB
| import logging | |
| import math | |
| import os | |
| import threading | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any, Iterable | |
| import numpy as np | |
| logger = logging.getLogger("neural_forge.models") | |
| PACKAGE_DIR = Path(__file__).resolve().parent | |
| PROJECT_DIR = PACKAGE_DIR.parent | |
| ASSETS_DIR = PROJECT_DIR / "assets" | |
| IRIS_DIAMETER_MM = 11.7 | |
| POSE_KEYPOINT_THRESHOLD = 0.4 | |
| def _env_bool(name: str, default: bool) -> bool: | |
| value = os.getenv(name) | |
| if value is None: | |
| return default | |
| return value.strip().lower() in {"1", "true", "yes", "on"} | |
| class Settings: | |
| qwen_model_id: str | |
| qwen_precision: str | |
| qwen_max_image_side: int | |
| qwen_max_new_tokens: int | |
| qwen_load_timeout_seconds: int | |
| face_task_path: str | |
| hand_task_path: str | |
| hand_label_swap: bool | |
| pose_mode: str | |
| pose_device: str | |
| max_frame_bytes: int | |
| max_frame_width: int | |
| session_ttl_seconds: int | |
| load_qwen_on_startup: bool | |
| worker_threads: int | |
| cors_origins: tuple[str, ...] | |
| def load_settings() -> Settings: | |
| cors_origins = tuple(origin.strip() for origin in os.getenv("CORS_ORIGINS", "*").split(",") if origin.strip()) | |
| return Settings( | |
| qwen_model_id=os.getenv("QWEN_MODEL_ID", "Qwen/Qwen3-VL-4B-Instruct"), | |
| qwen_precision=os.getenv("QWEN_PRECISION", "float16").strip().lower(), | |
| qwen_max_image_side=int(os.getenv("QWEN_MAX_IMAGE_SIDE", "448")), | |
| qwen_max_new_tokens=int(os.getenv("QWEN_MAX_NEW_TOKENS", "900")), | |
| qwen_load_timeout_seconds=int(os.getenv("QWEN_LOAD_TIMEOUT_SECONDS", "900")), | |
| face_task_path=os.getenv("FACE_TASK_PATH", str(ASSETS_DIR / "face_landmarker.task")), | |
| hand_task_path=os.getenv("HAND_TASK_PATH", str(ASSETS_DIR / "hand_landmarker.task")), | |
| hand_label_swap=_env_bool("HAND_LABEL_SWAP", True), | |
| pose_mode=os.getenv("POSE_MODE", "balanced"), | |
| pose_device=os.getenv("POSE_DEVICE", "cuda"), | |
| max_frame_bytes=int(os.getenv("MAX_FRAME_BYTES", "3000000")), | |
| max_frame_width=int(os.getenv("MAX_FRAME_WIDTH", "1280")), | |
| session_ttl_seconds=int(os.getenv("SESSION_TTL_SECONDS", "7200")), | |
| load_qwen_on_startup=_env_bool("LOAD_QWEN_ON_STARTUP", True), | |
| worker_threads=int(os.getenv("WORKER_THREADS", "4")), | |
| cors_origins=cors_origins or ("*",), | |
| ) | |
| SETTINGS = load_settings() | |
| class FaceIdx: | |
| RIGHT_EYE_OUTER = 33 | |
| RIGHT_EYE_INNER = 133 | |
| RIGHT_EYE_UPPER = 159 | |
| RIGHT_EYE_LOWER = 145 | |
| RIGHT_EYE_EAR = (33, 160, 158, 133, 153, 144) | |
| RIGHT_EYE_CONTOUR = (33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7) | |
| LEFT_EYE_OUTER = 263 | |
| LEFT_EYE_INNER = 362 | |
| LEFT_EYE_UPPER = 386 | |
| LEFT_EYE_LOWER = 374 | |
| LEFT_EYE_EAR = (362, 385, 387, 263, 373, 380) | |
| LEFT_EYE_CONTOUR = (263, 466, 388, 387, 386, 385, 384, 398, 362, 382, 381, 380, 374, 373, 390, 249) | |
| RIGHT_IRIS_CENTER = 468 | |
| RIGHT_IRIS_RING = (469, 470, 471, 472) | |
| LEFT_IRIS_CENTER = 473 | |
| LEFT_IRIS_RING = (474, 475, 476, 477) | |
| RIGHT_BROW = (70, 63, 105, 66, 107) | |
| LEFT_BROW = (300, 293, 334, 296, 336) | |
| RIGHT_BROW_MID = 105 | |
| LEFT_BROW_MID = 334 | |
| MOUTH_RIGHT = 61 | |
| MOUTH_LEFT = 291 | |
| UPPER_LIP_INNER = 13 | |
| LOWER_LIP_INNER = 14 | |
| UPPER_LIP_OUTER = 0 | |
| LOWER_LIP_OUTER = 17 | |
| LIPS_OUTER = (61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146) | |
| LIPS_INNER = (78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95) | |
| FOREHEAD_TOP = 10 | |
| FOREHEAD_CENTER = 151 | |
| GLABELLA = 9 | |
| NOSE_BRIDGE = 168 | |
| NOSE_TIP = 1 | |
| CHIN = 152 | |
| RIGHT_CHEEK = 205 | |
| LEFT_CHEEK = 425 | |
| RIGHT_FACE_EDGE = 234 | |
| LEFT_FACE_EDGE = 454 | |
| RIGHT_NOSE_ALA = 129 | |
| LEFT_NOSE_ALA = 358 | |
| FACE_OVAL = (10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, | |
| 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109) | |
| class HandIdx: | |
| WRIST = 0 | |
| THUMB_CMC = 1 | |
| THUMB_MCP = 2 | |
| THUMB_IP = 3 | |
| THUMB_TIP = 4 | |
| INDEX_MCP = 5 | |
| INDEX_PIP = 6 | |
| INDEX_DIP = 7 | |
| INDEX_TIP = 8 | |
| MIDDLE_MCP = 9 | |
| MIDDLE_PIP = 10 | |
| MIDDLE_DIP = 11 | |
| MIDDLE_TIP = 12 | |
| RING_MCP = 13 | |
| RING_PIP = 14 | |
| RING_DIP = 15 | |
| RING_TIP = 16 | |
| PINKY_MCP = 17 | |
| PINKY_PIP = 18 | |
| PINKY_DIP = 19 | |
| PINKY_TIP = 20 | |
| FINGERS = { | |
| "thumb": (1, 2, 3, 4), | |
| "index": (5, 6, 7, 8), | |
| "middle": (9, 10, 11, 12), | |
| "ring": (13, 14, 15, 16), | |
| "pinky": (17, 18, 19, 20), | |
| } | |
| class PoseIdx: | |
| NOSE = 0 | |
| LEFT_EYE = 1 | |
| RIGHT_EYE = 2 | |
| LEFT_EAR = 3 | |
| RIGHT_EAR = 4 | |
| LEFT_SHOULDER = 5 | |
| RIGHT_SHOULDER = 6 | |
| LEFT_ELBOW = 7 | |
| RIGHT_ELBOW = 8 | |
| LEFT_WRIST = 9 | |
| RIGHT_WRIST = 10 | |
| LEFT_HIP = 11 | |
| RIGHT_HIP = 12 | |
| LEFT_KNEE = 13 | |
| RIGHT_KNEE = 14 | |
| LEFT_ANKLE = 15 | |
| RIGHT_ANKLE = 16 | |
| LEFT_BIG_TOE = 17 | |
| LEFT_SMALL_TOE = 18 | |
| LEFT_HEEL = 19 | |
| RIGHT_BIG_TOE = 20 | |
| RIGHT_SMALL_TOE = 21 | |
| RIGHT_HEEL = 22 | |
| def _box_from_points(points: np.ndarray, width: int, height: int) -> tuple[int, int, int, int]: | |
| x_min = int(max(0, math.floor(float(points[:, 0].min())))) | |
| y_min = int(max(0, math.floor(float(points[:, 1].min())))) | |
| x_max = int(min(width - 1, math.ceil(float(points[:, 0].max())))) | |
| y_max = int(min(height - 1, math.ceil(float(points[:, 1].max())))) | |
| return x_min, y_min, x_max, y_max | |
| def _wrap_angle(angle: float) -> float: | |
| if angle > 90.0: | |
| return angle - 180.0 | |
| if angle < -90.0: | |
| return angle + 180.0 | |
| return angle | |
| def _head_pose_from_matrix(matrix: Any) -> tuple[float, float, float]: | |
| transform = np.asarray(matrix, dtype=np.float64) | |
| if transform.shape != (4, 4): | |
| return 0.0, 0.0, 0.0 | |
| rotation = transform[:3, :3].copy() | |
| for column in range(3): | |
| norm = float(np.linalg.norm(rotation[:, column])) | |
| if norm > 1e-9: | |
| rotation[:, column] /= norm | |
| horizontal = math.sqrt(rotation[0, 0] ** 2 + rotation[1, 0] ** 2) | |
| if horizontal > 1e-6: | |
| pitch = math.degrees(math.atan2(rotation[2, 1], rotation[2, 2])) | |
| yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal)) | |
| roll = math.degrees(math.atan2(rotation[1, 0], rotation[0, 0])) | |
| else: | |
| pitch = math.degrees(math.atan2(-rotation[1, 2], rotation[1, 1])) | |
| yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal)) | |
| roll = 0.0 | |
| return _wrap_angle(yaw), _wrap_angle(pitch), _wrap_angle(roll) | |
| class FaceData: | |
| points: np.ndarray | |
| blendshapes: dict[str, float] | |
| yaw: float | |
| pitch: float | |
| roll: float | |
| iris_diameter_px: float | |
| mm_per_px: float | |
| box: tuple[int, int, int, int] | |
| width_fraction: float | |
| def point(self, index: int) -> np.ndarray: | |
| return self.points[index, :2] | |
| def points_2d(self, indices: Iterable[int]) -> np.ndarray: | |
| return self.points[list(indices), :2] | |
| def blendshape(self, name: str) -> float: | |
| return float(self.blendshapes.get(name, 0.0)) | |
| def face_width_px(self) -> float: | |
| return float(np.linalg.norm(self.points[FaceIdx.RIGHT_FACE_EDGE, :2] - self.points[FaceIdx.LEFT_FACE_EDGE, :2])) | |
| class HandData: | |
| side: str | |
| points: np.ndarray | |
| score: float | |
| box: tuple[int, int, int, int] | |
| size_fraction: float | |
| def point(self, index: int) -> np.ndarray: | |
| return self.points[index, :2] | |
| def palm_width_px(self) -> float: | |
| return float(np.linalg.norm(self.points[HandIdx.INDEX_MCP, :2] - self.points[HandIdx.PINKY_MCP, :2])) | |
| def hand_length_px(self) -> float: | |
| return float(np.linalg.norm(self.points[HandIdx.WRIST, :2] - self.points[HandIdx.MIDDLE_MCP, :2])) | |
| class PoseData: | |
| points: np.ndarray | |
| scores: np.ndarray | |
| box: tuple[int, int, int, int] | |
| height_fraction: float | |
| def visible(self, index: int, threshold: float = POSE_KEYPOINT_THRESHOLD) -> bool: | |
| return bool(index < self.scores.shape[0] and float(self.scores[index]) >= threshold) | |
| def point(self, index: int) -> np.ndarray: | |
| return self.points[index, :2] | |
| class SessionTracker: | |
| def __init__(self) -> None: | |
| self._face_landmarker: Any = None | |
| self._hand_landmarker: Any = None | |
| self._last_timestamps: dict[str, int] = {"face": -1, "hands": -1} | |
| self._lock = threading.Lock() | |
| self._closed = False | |
| def _require_file(path: str, label: str) -> None: | |
| if not Path(path).is_file(): | |
| raise FileNotFoundError(f"{label} model file not found at {path}") | |
| def _create_face_landmarker(self) -> Any: | |
| from mediapipe.tasks import python as mp_python | |
| from mediapipe.tasks.python import vision | |
| self._require_file(SETTINGS.face_task_path, "Face landmarker") | |
| options = vision.FaceLandmarkerOptions( | |
| base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.face_task_path), | |
| running_mode=vision.RunningMode.VIDEO, | |
| num_faces=1, | |
| min_face_detection_confidence=0.5, | |
| min_face_presence_confidence=0.5, | |
| min_tracking_confidence=0.5, | |
| output_face_blendshapes=True, | |
| output_facial_transformation_matrixes=True, | |
| ) | |
| return vision.FaceLandmarker.create_from_options(options) | |
| def _create_hand_landmarker(self) -> Any: | |
| from mediapipe.tasks import python as mp_python | |
| from mediapipe.tasks.python import vision | |
| self._require_file(SETTINGS.hand_task_path, "Hand landmarker") | |
| options = vision.HandLandmarkerOptions( | |
| base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.hand_task_path), | |
| running_mode=vision.RunningMode.VIDEO, | |
| num_hands=2, | |
| min_hand_detection_confidence=0.5, | |
| min_hand_presence_confidence=0.5, | |
| min_tracking_confidence=0.5, | |
| ) | |
| return vision.HandLandmarker.create_from_options(options) | |
| def _next_timestamp(self, tracker_name: str, timestamp_ms: float) -> int: | |
| last = self._last_timestamps[tracker_name] | |
| timestamp = int(timestamp_ms) | |
| if timestamp <= last: | |
| timestamp = last + 1 | |
| self._last_timestamps[tracker_name] = timestamp | |
| return timestamp | |
| def process( | |
| self, | |
| image_bgr: np.ndarray, | |
| timestamp_ms: float, | |
| trackers: set[str], | |
| ) -> tuple[FaceData | None, dict[str, HandData], PoseData | None]: | |
| face: FaceData | None = None | |
| hands: dict[str, HandData] = {} | |
| pose: PoseData | None = None | |
| height, width = image_bgr.shape[:2] | |
| with self._lock: | |
| if self._closed: | |
| return face, hands, pose | |
| if "face" in trackers or "hands" in trackers: | |
| import mediapipe as mp | |
| rgb = np.ascontiguousarray(image_bgr[:, :, ::-1]) | |
| mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) | |
| if "face" in trackers: | |
| if self._face_landmarker is None: | |
| self._face_landmarker = self._create_face_landmarker() | |
| result = self._face_landmarker.detect_for_video(mp_image, self._next_timestamp("face", timestamp_ms)) | |
| face = self._convert_face(result, width, height) | |
| if "hands" in trackers: | |
| if self._hand_landmarker is None: | |
| self._hand_landmarker = self._create_hand_landmarker() | |
| result = self._hand_landmarker.detect_for_video(mp_image, self._next_timestamp("hands", timestamp_ms)) | |
| hands = self._convert_hands(result, width, height) | |
| if "pose" in trackers: | |
| pose = REGISTRY.detect_pose(image_bgr) | |
| return face, hands, pose | |
| def _convert_face(result: Any, width: int, height: int) -> FaceData | None: | |
| if not result.face_landmarks: | |
| return None | |
| landmarks = result.face_landmarks[0] | |
| if len(landmarks) < 478: | |
| return None | |
| points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32) | |
| blendshapes: dict[str, float] = {} | |
| if result.face_blendshapes: | |
| blendshapes = {category.category_name: float(category.score) for category in result.face_blendshapes[0]} | |
| yaw, pitch, roll = 0.0, 0.0, 0.0 | |
| if result.facial_transformation_matrixes: | |
| yaw, pitch, roll = _head_pose_from_matrix(result.facial_transformation_matrixes[0]) | |
| iris_diameters = [] | |
| for ring in (FaceIdx.RIGHT_IRIS_RING, FaceIdx.LEFT_IRIS_RING): | |
| ring_points = points[list(ring), :2] | |
| distances = [ | |
| float(np.linalg.norm(ring_points[i] - ring_points[j])) | |
| for i in range(len(ring_points)) | |
| for j in range(i + 1, len(ring_points)) | |
| ] | |
| iris_diameters.append(max(distances)) | |
| iris_diameter = float(np.mean(iris_diameters)) | |
| mm_per_px = IRIS_DIAMETER_MM / iris_diameter if iris_diameter > 1.0 else 0.0 | |
| box = _box_from_points(points, width, height) | |
| width_fraction = float((points[:, 0].max() - points[:, 0].min()) / max(1, width)) | |
| return FaceData( | |
| points=points, | |
| blendshapes=blendshapes, | |
| yaw=yaw, | |
| pitch=pitch, | |
| roll=roll, | |
| iris_diameter_px=iris_diameter, | |
| mm_per_px=mm_per_px, | |
| box=box, | |
| width_fraction=width_fraction, | |
| ) | |
| def _convert_hands(result: Any, width: int, height: int) -> dict[str, HandData]: | |
| hands: dict[str, HandData] = {} | |
| if not result.hand_landmarks: | |
| return hands | |
| for landmarks, handedness in zip(result.hand_landmarks, result.handedness): | |
| if not handedness: | |
| continue | |
| label = handedness[0].category_name.strip().lower() | |
| score = float(handedness[0].score) | |
| if SETTINGS.hand_label_swap: | |
| side = "left" if label == "right" else "right" | |
| else: | |
| side = "right" if label == "right" else "left" | |
| points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32) | |
| box = _box_from_points(points, width, height) | |
| size_fraction = float(max(box[2] - box[0], box[3] - box[1]) / max(1, width)) | |
| if side in hands and hands[side].score >= score: | |
| continue | |
| hands[side] = HandData(side=side, points=points, score=score, box=box, size_fraction=size_fraction) | |
| return hands | |
| def close(self) -> None: | |
| with self._lock: | |
| self._closed = True | |
| for landmarker in (self._face_landmarker, self._hand_landmarker): | |
| if landmarker is not None: | |
| try: | |
| landmarker.close() | |
| except Exception: | |
| logger.warning("Failed to close landmarker", exc_info=True) | |
| self._face_landmarker = None | |
| self._hand_landmarker = None | |
| class ModelRegistry: | |
| def __init__(self) -> None: | |
| self.status: dict[str, str] = {"pose": "not_loaded", "qwen": "not_loaded"} | |
| self.errors: dict[str, str] = {} | |
| self._pose_model: Any = None | |
| self._pose_init_lock = threading.Lock() | |
| self._pose_run_lock = threading.Lock() | |
| self._qwen_model: Any = None | |
| self._qwen_processor: Any = None | |
| self._qwen_load_lock = threading.Lock() | |
| self._qwen_run_lock = threading.Lock() | |
| self._qwen_finished = threading.Event() | |
| self._qwen_started = False | |
| self._token_cache: dict[str, int] = {} | |
| def warm_up(self) -> None: | |
| try: | |
| self.pose_model() | |
| self.detect_pose(np.zeros((480, 640, 3), dtype=np.uint8)) | |
| except Exception as error: | |
| self.status["pose"] = "failed" | |
| self.errors["pose"] = str(error) | |
| logger.exception("Pose model failed to load") | |
| if SETTINGS.load_qwen_on_startup: | |
| self._qwen_started = True | |
| self.load_qwen() | |
| def pose_model(self) -> Any: | |
| with self._pose_init_lock: | |
| if self._pose_model is None: | |
| self.status["pose"] = "loading" | |
| from rtmlib import Wholebody | |
| device = SETTINGS.pose_device | |
| if device == "cuda": | |
| try: | |
| import onnxruntime | |
| if "CUDAExecutionProvider" not in onnxruntime.get_available_providers(): | |
| logger.warning("CUDA provider unavailable for onnxruntime, pose runs on CPU") | |
| device = "cpu" | |
| except Exception: | |
| device = "cpu" | |
| self._pose_model = Wholebody( | |
| to_openpose=False, | |
| mode=SETTINGS.pose_mode, | |
| backend="onnxruntime", | |
| device=device, | |
| ) | |
| self.status["pose"] = "ready" | |
| logger.info("Pose model ready on %s", device) | |
| return self._pose_model | |
| def detect_pose(self, image_bgr: np.ndarray) -> PoseData | None: | |
| model = self.pose_model() | |
| with self._pose_run_lock: | |
| keypoints, scores = model(image_bgr) | |
| if keypoints is None or scores is None: | |
| return None | |
| keypoints = np.asarray(keypoints, dtype=np.float32) | |
| scores = np.asarray(scores, dtype=np.float32) | |
| if keypoints.size == 0: | |
| return None | |
| if keypoints.ndim == 2: | |
| keypoints = keypoints[None, ...] | |
| scores = scores[None, ...] | |
| height, width = image_bgr.shape[:2] | |
| best_pose: PoseData | None = None | |
| best_area = 0.0 | |
| for person_points, person_scores in zip(keypoints, scores): | |
| if float(np.mean(person_scores[:17])) < 0.2: | |
| continue | |
| body_points = person_points[:23] | |
| visible_points = body_points[person_scores[:23] >= 0.3] | |
| if visible_points.shape[0] < 4: | |
| continue | |
| box = _box_from_points(visible_points, width, height) | |
| area = float((box[2] - box[0]) * (box[3] - box[1])) | |
| if area > best_area: | |
| best_area = area | |
| best_pose = PoseData( | |
| points=person_points.astype(np.float32), | |
| scores=person_scores.astype(np.float32), | |
| box=box, | |
| height_fraction=float((box[3] - box[1]) / max(1, height)), | |
| ) | |
| return best_pose | |
| def load_qwen(self) -> None: | |
| with self._qwen_load_lock: | |
| if self._qwen_model is not None or self.status["qwen"] == "failed": | |
| self._qwen_finished.set() | |
| return | |
| self.status["qwen"] = "loading" | |
| try: | |
| self._load_qwen_weights() | |
| self.status["qwen"] = "ready" | |
| logger.info("Qwen model ready: %s", SETTINGS.qwen_model_id) | |
| except Exception as error: | |
| self.status["qwen"] = "failed" | |
| self.errors["qwen"] = str(error) | |
| logger.exception("Qwen model failed to load") | |
| finally: | |
| self._qwen_finished.set() | |
| def _load_qwen_weights(self) -> None: | |
| import torch | |
| from transformers import AutoProcessor, Qwen3VLForConditionalGeneration | |
| token = os.getenv("HF_TOKEN") or None | |
| model_kwargs: dict[str, Any] = {"device_map": "auto", "low_cpu_mem_usage": True, "token": token} | |
| if torch.cuda.is_available(): | |
| if SETTINGS.qwen_precision == "4bit": | |
| from transformers import BitsAndBytesConfig | |
| model_kwargs["quantization_config"] = BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_quant_type="nf4", | |
| bnb_4bit_compute_dtype=torch.float16, | |
| bnb_4bit_use_double_quant=True, | |
| ) | |
| model_kwargs["dtype"] = torch.float16 | |
| elif SETTINGS.qwen_precision == "bfloat16": | |
| model_kwargs["dtype"] = torch.bfloat16 | |
| else: | |
| model_kwargs["dtype"] = torch.float16 | |
| else: | |
| logger.warning("CUDA not available, Qwen will run on CPU and be very slow") | |
| model_kwargs["dtype"] = torch.float32 | |
| processor = AutoProcessor.from_pretrained(SETTINGS.qwen_model_id, token=token) | |
| try: | |
| model = Qwen3VLForConditionalGeneration.from_pretrained( | |
| SETTINGS.qwen_model_id, attn_implementation="sdpa", **model_kwargs | |
| ) | |
| except (ValueError, ImportError, TypeError): | |
| logger.warning("SDPA attention unavailable, loading with default attention", exc_info=True) | |
| model = Qwen3VLForConditionalGeneration.from_pretrained(SETTINGS.qwen_model_id, **model_kwargs) | |
| model.eval() | |
| self._qwen_processor = processor | |
| self._qwen_model = model | |
| def wait_for_qwen(self, timeout_seconds: float) -> bool: | |
| if self.status["qwen"] == "ready": | |
| return True | |
| if self.status["qwen"] == "failed": | |
| return False | |
| if not self._qwen_started: | |
| self._qwen_started = True | |
| threading.Thread(target=self.load_qwen, name="qwen-loader", daemon=True).start() | |
| self._qwen_finished.wait(timeout_seconds) | |
| return self.status["qwen"] == "ready" | |
| def _prepare_inputs(self, messages: list[dict[str, Any]]) -> Any: | |
| inputs = self._qwen_processor.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| return_dict=True, | |
| return_tensors="pt", | |
| ) | |
| inputs = inputs.to(self._qwen_model.device) | |
| if "pixel_values" in inputs: | |
| inputs["pixel_values"] = inputs["pixel_values"].to(self._qwen_model.dtype) | |
| return inputs | |
| def qwen_generate(self, messages: list[dict[str, Any]], max_new_tokens: int | None = None) -> str: | |
| if self._qwen_model is None: | |
| raise RuntimeError("Qwen model is not loaded") | |
| import torch | |
| with self._qwen_run_lock: | |
| inputs = self._prepare_inputs(messages) | |
| with torch.inference_mode(): | |
| output_ids = self._qwen_model.generate( | |
| **inputs, | |
| max_new_tokens=max_new_tokens or SETTINGS.qwen_max_new_tokens, | |
| do_sample=False, | |
| repetition_penalty=1.05, | |
| ) | |
| prompt_length = inputs["input_ids"].shape[1] | |
| generated = output_ids[:, prompt_length:] | |
| text = self._qwen_processor.batch_decode( | |
| generated, skip_special_tokens=True, clean_up_tokenization_spaces=False | |
| )[0] | |
| return text.strip() | |
| def _letter_token_id(self, letter: str) -> int: | |
| if letter not in self._token_cache: | |
| token_ids = self._qwen_processor.tokenizer.encode(letter, add_special_tokens=False) | |
| if not token_ids: | |
| raise ValueError(f"Cannot tokenize option letter {letter}") | |
| self._token_cache[letter] = int(token_ids[0]) | |
| return self._token_cache[letter] | |
| def qwen_option_probabilities(self, messages: list[dict[str, Any]], letters: list[str]) -> list[float]: | |
| if self._qwen_model is None: | |
| raise RuntimeError("Qwen model is not loaded") | |
| import torch | |
| with self._qwen_run_lock: | |
| inputs = self._prepare_inputs(messages) | |
| with torch.inference_mode(): | |
| outputs = self._qwen_model(**inputs) | |
| logits = outputs.logits[0, -1, :].float() | |
| token_ids = [self._letter_token_id(letter) for letter in letters] | |
| selected = logits[token_ids] | |
| probabilities = torch.softmax(selected, dim=0).cpu().tolist() | |
| return [float(value) for value in probabilities] | |
| def gpu_info(self) -> dict[str, Any]: | |
| try: | |
| import torch | |
| if not torch.cuda.is_available(): | |
| return {"available": False} | |
| properties = torch.cuda.get_device_properties(0) | |
| return { | |
| "available": True, | |
| "name": properties.name, | |
| "total_memory_gb": round(properties.total_memory / 1e9, 2), | |
| "allocated_memory_gb": round(torch.cuda.memory_allocated(0) / 1e9, 2), | |
| } | |
| except Exception: | |
| return {"available": False} | |
| REGISTRY = ModelRegistry() | |