import logging import math import os import threading from dataclasses import dataclass from pathlib import Path from typing import Any, Iterable import numpy as np logger = logging.getLogger("neural_forge.models") PACKAGE_DIR = Path(__file__).resolve().parent PROJECT_DIR = PACKAGE_DIR.parent ASSETS_DIR = PROJECT_DIR / "assets" IRIS_DIAMETER_MM = 11.7 POSE_KEYPOINT_THRESHOLD = 0.4 def _env_bool(name: str, default: bool) -> bool: value = os.getenv(name) if value is None: return default return value.strip().lower() in {"1", "true", "yes", "on"} @dataclass(frozen=True) class Settings: qwen_model_id: str qwen_precision: str qwen_max_image_side: int qwen_max_new_tokens: int qwen_load_timeout_seconds: int face_task_path: str hand_task_path: str hand_label_swap: bool pose_mode: str pose_device: str max_frame_bytes: int max_frame_width: int session_ttl_seconds: int load_qwen_on_startup: bool worker_threads: int cors_origins: tuple[str, ...] def load_settings() -> Settings: cors_origins = tuple(origin.strip() for origin in os.getenv("CORS_ORIGINS", "*").split(",") if origin.strip()) return Settings( qwen_model_id=os.getenv("QWEN_MODEL_ID", "Qwen/Qwen3-VL-4B-Instruct"), qwen_precision=os.getenv("QWEN_PRECISION", "float16").strip().lower(), qwen_max_image_side=int(os.getenv("QWEN_MAX_IMAGE_SIDE", "448")), qwen_max_new_tokens=int(os.getenv("QWEN_MAX_NEW_TOKENS", "900")), qwen_load_timeout_seconds=int(os.getenv("QWEN_LOAD_TIMEOUT_SECONDS", "900")), face_task_path=os.getenv("FACE_TASK_PATH", str(ASSETS_DIR / "face_landmarker.task")), hand_task_path=os.getenv("HAND_TASK_PATH", str(ASSETS_DIR / "hand_landmarker.task")), hand_label_swap=_env_bool("HAND_LABEL_SWAP", True), pose_mode=os.getenv("POSE_MODE", "balanced"), pose_device=os.getenv("POSE_DEVICE", "cuda"), max_frame_bytes=int(os.getenv("MAX_FRAME_BYTES", "3000000")), max_frame_width=int(os.getenv("MAX_FRAME_WIDTH", "1280")), session_ttl_seconds=int(os.getenv("SESSION_TTL_SECONDS", "7200")), load_qwen_on_startup=_env_bool("LOAD_QWEN_ON_STARTUP", True), worker_threads=int(os.getenv("WORKER_THREADS", "4")), cors_origins=cors_origins or ("*",), ) SETTINGS = load_settings() class FaceIdx: RIGHT_EYE_OUTER = 33 RIGHT_EYE_INNER = 133 RIGHT_EYE_UPPER = 159 RIGHT_EYE_LOWER = 145 RIGHT_EYE_EAR = (33, 160, 158, 133, 153, 144) RIGHT_EYE_CONTOUR = (33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7) LEFT_EYE_OUTER = 263 LEFT_EYE_INNER = 362 LEFT_EYE_UPPER = 386 LEFT_EYE_LOWER = 374 LEFT_EYE_EAR = (362, 385, 387, 263, 373, 380) LEFT_EYE_CONTOUR = (263, 466, 388, 387, 386, 385, 384, 398, 362, 382, 381, 380, 374, 373, 390, 249) RIGHT_IRIS_CENTER = 468 RIGHT_IRIS_RING = (469, 470, 471, 472) LEFT_IRIS_CENTER = 473 LEFT_IRIS_RING = (474, 475, 476, 477) RIGHT_BROW = (70, 63, 105, 66, 107) LEFT_BROW = (300, 293, 334, 296, 336) RIGHT_BROW_MID = 105 LEFT_BROW_MID = 334 MOUTH_RIGHT = 61 MOUTH_LEFT = 291 UPPER_LIP_INNER = 13 LOWER_LIP_INNER = 14 UPPER_LIP_OUTER = 0 LOWER_LIP_OUTER = 17 LIPS_OUTER = (61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146) LIPS_INNER = (78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95) FOREHEAD_TOP = 10 FOREHEAD_CENTER = 151 GLABELLA = 9 NOSE_BRIDGE = 168 NOSE_TIP = 1 CHIN = 152 RIGHT_CHEEK = 205 LEFT_CHEEK = 425 RIGHT_FACE_EDGE = 234 LEFT_FACE_EDGE = 454 RIGHT_NOSE_ALA = 129 LEFT_NOSE_ALA = 358 FACE_OVAL = (10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377, 152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109) class HandIdx: WRIST = 0 THUMB_CMC = 1 THUMB_MCP = 2 THUMB_IP = 3 THUMB_TIP = 4 INDEX_MCP = 5 INDEX_PIP = 6 INDEX_DIP = 7 INDEX_TIP = 8 MIDDLE_MCP = 9 MIDDLE_PIP = 10 MIDDLE_DIP = 11 MIDDLE_TIP = 12 RING_MCP = 13 RING_PIP = 14 RING_DIP = 15 RING_TIP = 16 PINKY_MCP = 17 PINKY_PIP = 18 PINKY_DIP = 19 PINKY_TIP = 20 FINGERS = { "thumb": (1, 2, 3, 4), "index": (5, 6, 7, 8), "middle": (9, 10, 11, 12), "ring": (13, 14, 15, 16), "pinky": (17, 18, 19, 20), } class PoseIdx: NOSE = 0 LEFT_EYE = 1 RIGHT_EYE = 2 LEFT_EAR = 3 RIGHT_EAR = 4 LEFT_SHOULDER = 5 RIGHT_SHOULDER = 6 LEFT_ELBOW = 7 RIGHT_ELBOW = 8 LEFT_WRIST = 9 RIGHT_WRIST = 10 LEFT_HIP = 11 RIGHT_HIP = 12 LEFT_KNEE = 13 RIGHT_KNEE = 14 LEFT_ANKLE = 15 RIGHT_ANKLE = 16 LEFT_BIG_TOE = 17 LEFT_SMALL_TOE = 18 LEFT_HEEL = 19 RIGHT_BIG_TOE = 20 RIGHT_SMALL_TOE = 21 RIGHT_HEEL = 22 def _box_from_points(points: np.ndarray, width: int, height: int) -> tuple[int, int, int, int]: x_min = int(max(0, math.floor(float(points[:, 0].min())))) y_min = int(max(0, math.floor(float(points[:, 1].min())))) x_max = int(min(width - 1, math.ceil(float(points[:, 0].max())))) y_max = int(min(height - 1, math.ceil(float(points[:, 1].max())))) return x_min, y_min, x_max, y_max def _wrap_angle(angle: float) -> float: if angle > 90.0: return angle - 180.0 if angle < -90.0: return angle + 180.0 return angle def _head_pose_from_matrix(matrix: Any) -> tuple[float, float, float]: transform = np.asarray(matrix, dtype=np.float64) if transform.shape != (4, 4): return 0.0, 0.0, 0.0 rotation = transform[:3, :3].copy() for column in range(3): norm = float(np.linalg.norm(rotation[:, column])) if norm > 1e-9: rotation[:, column] /= norm horizontal = math.sqrt(rotation[0, 0] ** 2 + rotation[1, 0] ** 2) if horizontal > 1e-6: pitch = math.degrees(math.atan2(rotation[2, 1], rotation[2, 2])) yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal)) roll = math.degrees(math.atan2(rotation[1, 0], rotation[0, 0])) else: pitch = math.degrees(math.atan2(-rotation[1, 2], rotation[1, 1])) yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal)) roll = 0.0 return _wrap_angle(yaw), _wrap_angle(pitch), _wrap_angle(roll) @dataclass class FaceData: points: np.ndarray blendshapes: dict[str, float] yaw: float pitch: float roll: float iris_diameter_px: float mm_per_px: float box: tuple[int, int, int, int] width_fraction: float def point(self, index: int) -> np.ndarray: return self.points[index, :2] def points_2d(self, indices: Iterable[int]) -> np.ndarray: return self.points[list(indices), :2] def blendshape(self, name: str) -> float: return float(self.blendshapes.get(name, 0.0)) def face_width_px(self) -> float: return float(np.linalg.norm(self.points[FaceIdx.RIGHT_FACE_EDGE, :2] - self.points[FaceIdx.LEFT_FACE_EDGE, :2])) @dataclass class HandData: side: str points: np.ndarray score: float box: tuple[int, int, int, int] size_fraction: float def point(self, index: int) -> np.ndarray: return self.points[index, :2] def palm_width_px(self) -> float: return float(np.linalg.norm(self.points[HandIdx.INDEX_MCP, :2] - self.points[HandIdx.PINKY_MCP, :2])) def hand_length_px(self) -> float: return float(np.linalg.norm(self.points[HandIdx.WRIST, :2] - self.points[HandIdx.MIDDLE_MCP, :2])) @dataclass class PoseData: points: np.ndarray scores: np.ndarray box: tuple[int, int, int, int] height_fraction: float def visible(self, index: int, threshold: float = POSE_KEYPOINT_THRESHOLD) -> bool: return bool(index < self.scores.shape[0] and float(self.scores[index]) >= threshold) def point(self, index: int) -> np.ndarray: return self.points[index, :2] class SessionTracker: def __init__(self) -> None: self._face_landmarker: Any = None self._hand_landmarker: Any = None self._last_timestamps: dict[str, int] = {"face": -1, "hands": -1} self._lock = threading.Lock() self._closed = False @staticmethod def _require_file(path: str, label: str) -> None: if not Path(path).is_file(): raise FileNotFoundError(f"{label} model file not found at {path}") def _create_face_landmarker(self) -> Any: from mediapipe.tasks import python as mp_python from mediapipe.tasks.python import vision self._require_file(SETTINGS.face_task_path, "Face landmarker") options = vision.FaceLandmarkerOptions( base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.face_task_path), running_mode=vision.RunningMode.VIDEO, num_faces=1, min_face_detection_confidence=0.5, min_face_presence_confidence=0.5, min_tracking_confidence=0.5, output_face_blendshapes=True, output_facial_transformation_matrixes=True, ) return vision.FaceLandmarker.create_from_options(options) def _create_hand_landmarker(self) -> Any: from mediapipe.tasks import python as mp_python from mediapipe.tasks.python import vision self._require_file(SETTINGS.hand_task_path, "Hand landmarker") options = vision.HandLandmarkerOptions( base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.hand_task_path), running_mode=vision.RunningMode.VIDEO, num_hands=2, min_hand_detection_confidence=0.5, min_hand_presence_confidence=0.5, min_tracking_confidence=0.5, ) return vision.HandLandmarker.create_from_options(options) def _next_timestamp(self, tracker_name: str, timestamp_ms: float) -> int: last = self._last_timestamps[tracker_name] timestamp = int(timestamp_ms) if timestamp <= last: timestamp = last + 1 self._last_timestamps[tracker_name] = timestamp return timestamp def process( self, image_bgr: np.ndarray, timestamp_ms: float, trackers: set[str], ) -> tuple[FaceData | None, dict[str, HandData], PoseData | None]: face: FaceData | None = None hands: dict[str, HandData] = {} pose: PoseData | None = None height, width = image_bgr.shape[:2] with self._lock: if self._closed: return face, hands, pose if "face" in trackers or "hands" in trackers: import mediapipe as mp rgb = np.ascontiguousarray(image_bgr[:, :, ::-1]) mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb) if "face" in trackers: if self._face_landmarker is None: self._face_landmarker = self._create_face_landmarker() result = self._face_landmarker.detect_for_video(mp_image, self._next_timestamp("face", timestamp_ms)) face = self._convert_face(result, width, height) if "hands" in trackers: if self._hand_landmarker is None: self._hand_landmarker = self._create_hand_landmarker() result = self._hand_landmarker.detect_for_video(mp_image, self._next_timestamp("hands", timestamp_ms)) hands = self._convert_hands(result, width, height) if "pose" in trackers: pose = REGISTRY.detect_pose(image_bgr) return face, hands, pose @staticmethod def _convert_face(result: Any, width: int, height: int) -> FaceData | None: if not result.face_landmarks: return None landmarks = result.face_landmarks[0] if len(landmarks) < 478: return None points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32) blendshapes: dict[str, float] = {} if result.face_blendshapes: blendshapes = {category.category_name: float(category.score) for category in result.face_blendshapes[0]} yaw, pitch, roll = 0.0, 0.0, 0.0 if result.facial_transformation_matrixes: yaw, pitch, roll = _head_pose_from_matrix(result.facial_transformation_matrixes[0]) iris_diameters = [] for ring in (FaceIdx.RIGHT_IRIS_RING, FaceIdx.LEFT_IRIS_RING): ring_points = points[list(ring), :2] distances = [ float(np.linalg.norm(ring_points[i] - ring_points[j])) for i in range(len(ring_points)) for j in range(i + 1, len(ring_points)) ] iris_diameters.append(max(distances)) iris_diameter = float(np.mean(iris_diameters)) mm_per_px = IRIS_DIAMETER_MM / iris_diameter if iris_diameter > 1.0 else 0.0 box = _box_from_points(points, width, height) width_fraction = float((points[:, 0].max() - points[:, 0].min()) / max(1, width)) return FaceData( points=points, blendshapes=blendshapes, yaw=yaw, pitch=pitch, roll=roll, iris_diameter_px=iris_diameter, mm_per_px=mm_per_px, box=box, width_fraction=width_fraction, ) @staticmethod def _convert_hands(result: Any, width: int, height: int) -> dict[str, HandData]: hands: dict[str, HandData] = {} if not result.hand_landmarks: return hands for landmarks, handedness in zip(result.hand_landmarks, result.handedness): if not handedness: continue label = handedness[0].category_name.strip().lower() score = float(handedness[0].score) if SETTINGS.hand_label_swap: side = "left" if label == "right" else "right" else: side = "right" if label == "right" else "left" points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32) box = _box_from_points(points, width, height) size_fraction = float(max(box[2] - box[0], box[3] - box[1]) / max(1, width)) if side in hands and hands[side].score >= score: continue hands[side] = HandData(side=side, points=points, score=score, box=box, size_fraction=size_fraction) return hands def close(self) -> None: with self._lock: self._closed = True for landmarker in (self._face_landmarker, self._hand_landmarker): if landmarker is not None: try: landmarker.close() except Exception: logger.warning("Failed to close landmarker", exc_info=True) self._face_landmarker = None self._hand_landmarker = None class ModelRegistry: def __init__(self) -> None: self.status: dict[str, str] = {"pose": "not_loaded", "qwen": "not_loaded"} self.errors: dict[str, str] = {} self._pose_model: Any = None self._pose_init_lock = threading.Lock() self._pose_run_lock = threading.Lock() self._qwen_model: Any = None self._qwen_processor: Any = None self._qwen_load_lock = threading.Lock() self._qwen_run_lock = threading.Lock() self._qwen_finished = threading.Event() self._qwen_started = False self._token_cache: dict[str, int] = {} def warm_up(self) -> None: try: self.pose_model() self.detect_pose(np.zeros((480, 640, 3), dtype=np.uint8)) except Exception as error: self.status["pose"] = "failed" self.errors["pose"] = str(error) logger.exception("Pose model failed to load") if SETTINGS.load_qwen_on_startup: self._qwen_started = True self.load_qwen() def pose_model(self) -> Any: with self._pose_init_lock: if self._pose_model is None: self.status["pose"] = "loading" from rtmlib import Wholebody device = SETTINGS.pose_device if device == "cuda": try: import onnxruntime if "CUDAExecutionProvider" not in onnxruntime.get_available_providers(): logger.warning("CUDA provider unavailable for onnxruntime, pose runs on CPU") device = "cpu" except Exception: device = "cpu" self._pose_model = Wholebody( to_openpose=False, mode=SETTINGS.pose_mode, backend="onnxruntime", device=device, ) self.status["pose"] = "ready" logger.info("Pose model ready on %s", device) return self._pose_model def detect_pose(self, image_bgr: np.ndarray) -> PoseData | None: model = self.pose_model() with self._pose_run_lock: keypoints, scores = model(image_bgr) if keypoints is None or scores is None: return None keypoints = np.asarray(keypoints, dtype=np.float32) scores = np.asarray(scores, dtype=np.float32) if keypoints.size == 0: return None if keypoints.ndim == 2: keypoints = keypoints[None, ...] scores = scores[None, ...] height, width = image_bgr.shape[:2] best_pose: PoseData | None = None best_area = 0.0 for person_points, person_scores in zip(keypoints, scores): if float(np.mean(person_scores[:17])) < 0.2: continue body_points = person_points[:23] visible_points = body_points[person_scores[:23] >= 0.3] if visible_points.shape[0] < 4: continue box = _box_from_points(visible_points, width, height) area = float((box[2] - box[0]) * (box[3] - box[1])) if area > best_area: best_area = area best_pose = PoseData( points=person_points.astype(np.float32), scores=person_scores.astype(np.float32), box=box, height_fraction=float((box[3] - box[1]) / max(1, height)), ) return best_pose def load_qwen(self) -> None: with self._qwen_load_lock: if self._qwen_model is not None or self.status["qwen"] == "failed": self._qwen_finished.set() return self.status["qwen"] = "loading" try: self._load_qwen_weights() self.status["qwen"] = "ready" logger.info("Qwen model ready: %s", SETTINGS.qwen_model_id) except Exception as error: self.status["qwen"] = "failed" self.errors["qwen"] = str(error) logger.exception("Qwen model failed to load") finally: self._qwen_finished.set() def _load_qwen_weights(self) -> None: import torch from transformers import AutoProcessor, Qwen3VLForConditionalGeneration token = os.getenv("HF_TOKEN") or None model_kwargs: dict[str, Any] = {"device_map": "auto", "low_cpu_mem_usage": True, "token": token} if torch.cuda.is_available(): if SETTINGS.qwen_precision == "4bit": from transformers import BitsAndBytesConfig model_kwargs["quantization_config"] = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16, bnb_4bit_use_double_quant=True, ) model_kwargs["dtype"] = torch.float16 elif SETTINGS.qwen_precision == "bfloat16": model_kwargs["dtype"] = torch.bfloat16 else: model_kwargs["dtype"] = torch.float16 else: logger.warning("CUDA not available, Qwen will run on CPU and be very slow") model_kwargs["dtype"] = torch.float32 processor = AutoProcessor.from_pretrained(SETTINGS.qwen_model_id, token=token) try: model = Qwen3VLForConditionalGeneration.from_pretrained( SETTINGS.qwen_model_id, attn_implementation="sdpa", **model_kwargs ) except (ValueError, ImportError, TypeError): logger.warning("SDPA attention unavailable, loading with default attention", exc_info=True) model = Qwen3VLForConditionalGeneration.from_pretrained(SETTINGS.qwen_model_id, **model_kwargs) model.eval() self._qwen_processor = processor self._qwen_model = model def wait_for_qwen(self, timeout_seconds: float) -> bool: if self.status["qwen"] == "ready": return True if self.status["qwen"] == "failed": return False if not self._qwen_started: self._qwen_started = True threading.Thread(target=self.load_qwen, name="qwen-loader", daemon=True).start() self._qwen_finished.wait(timeout_seconds) return self.status["qwen"] == "ready" def _prepare_inputs(self, messages: list[dict[str, Any]]) -> Any: inputs = self._qwen_processor.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, return_dict=True, return_tensors="pt", ) inputs = inputs.to(self._qwen_model.device) if "pixel_values" in inputs: inputs["pixel_values"] = inputs["pixel_values"].to(self._qwen_model.dtype) return inputs def qwen_generate(self, messages: list[dict[str, Any]], max_new_tokens: int | None = None) -> str: if self._qwen_model is None: raise RuntimeError("Qwen model is not loaded") import torch with self._qwen_run_lock: inputs = self._prepare_inputs(messages) with torch.inference_mode(): output_ids = self._qwen_model.generate( **inputs, max_new_tokens=max_new_tokens or SETTINGS.qwen_max_new_tokens, do_sample=False, repetition_penalty=1.05, ) prompt_length = inputs["input_ids"].shape[1] generated = output_ids[:, prompt_length:] text = self._qwen_processor.batch_decode( generated, skip_special_tokens=True, clean_up_tokenization_spaces=False )[0] return text.strip() def _letter_token_id(self, letter: str) -> int: if letter not in self._token_cache: token_ids = self._qwen_processor.tokenizer.encode(letter, add_special_tokens=False) if not token_ids: raise ValueError(f"Cannot tokenize option letter {letter}") self._token_cache[letter] = int(token_ids[0]) return self._token_cache[letter] def qwen_option_probabilities(self, messages: list[dict[str, Any]], letters: list[str]) -> list[float]: if self._qwen_model is None: raise RuntimeError("Qwen model is not loaded") import torch with self._qwen_run_lock: inputs = self._prepare_inputs(messages) with torch.inference_mode(): outputs = self._qwen_model(**inputs) logits = outputs.logits[0, -1, :].float() token_ids = [self._letter_token_id(letter) for letter in letters] selected = logits[token_ids] probabilities = torch.softmax(selected, dim=0).cpu().tolist() return [float(value) for value in probabilities] def gpu_info(self) -> dict[str, Any]: try: import torch if not torch.cuda.is_available(): return {"available": False} properties = torch.cuda.get_device_properties(0) return { "available": True, "name": properties.name, "total_memory_gb": round(properties.total_memory / 1e9, 2), "allocated_memory_gb": round(torch.cuda.memory_allocated(0) / 1e9, 2), } except Exception: return {"available": False} REGISTRY = ModelRegistry()