drrobot9's picture
Neural Forge backend
ceeb274 verified
Raw History Blame
24.9 kB
import logging
import math
import os
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Iterable
import numpy as np
logger = logging.getLogger("neural_forge.models")
PACKAGE_DIR = Path(__file__).resolve().parent
PROJECT_DIR = PACKAGE_DIR.parent
ASSETS_DIR = PROJECT_DIR / "assets"
IRIS_DIAMETER_MM = 11.7
POSE_KEYPOINT_THRESHOLD = 0.4
def _env_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
@dataclass(frozen=True)
class Settings:
qwen_model_id: str
qwen_precision: str
qwen_max_image_side: int
qwen_max_new_tokens: int
qwen_load_timeout_seconds: int
face_task_path: str
hand_task_path: str
hand_label_swap: bool
pose_mode: str
pose_device: str
max_frame_bytes: int
max_frame_width: int
session_ttl_seconds: int
load_qwen_on_startup: bool
worker_threads: int
cors_origins: tuple[str, ...]
def load_settings() -> Settings:
cors_origins = tuple(origin.strip() for origin in os.getenv("CORS_ORIGINS", "*").split(",") if origin.strip())
return Settings(
qwen_model_id=os.getenv("QWEN_MODEL_ID", "Qwen/Qwen3-VL-4B-Instruct"),
qwen_precision=os.getenv("QWEN_PRECISION", "float16").strip().lower(),
qwen_max_image_side=int(os.getenv("QWEN_MAX_IMAGE_SIDE", "448")),
qwen_max_new_tokens=int(os.getenv("QWEN_MAX_NEW_TOKENS", "900")),
qwen_load_timeout_seconds=int(os.getenv("QWEN_LOAD_TIMEOUT_SECONDS", "900")),
face_task_path=os.getenv("FACE_TASK_PATH", str(ASSETS_DIR / "face_landmarker.task")),
hand_task_path=os.getenv("HAND_TASK_PATH", str(ASSETS_DIR / "hand_landmarker.task")),
hand_label_swap=_env_bool("HAND_LABEL_SWAP", True),
pose_mode=os.getenv("POSE_MODE", "balanced"),
pose_device=os.getenv("POSE_DEVICE", "cuda"),
max_frame_bytes=int(os.getenv("MAX_FRAME_BYTES", "3000000")),
max_frame_width=int(os.getenv("MAX_FRAME_WIDTH", "1280")),
session_ttl_seconds=int(os.getenv("SESSION_TTL_SECONDS", "7200")),
load_qwen_on_startup=_env_bool("LOAD_QWEN_ON_STARTUP", True),
worker_threads=int(os.getenv("WORKER_THREADS", "4")),
cors_origins=cors_origins or ("*",),
)
SETTINGS = load_settings()
class FaceIdx:
RIGHT_EYE_OUTER = 33
RIGHT_EYE_INNER = 133
RIGHT_EYE_UPPER = 159
RIGHT_EYE_LOWER = 145
RIGHT_EYE_EAR = (33, 160, 158, 133, 153, 144)
RIGHT_EYE_CONTOUR = (33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7)
LEFT_EYE_OUTER = 263
LEFT_EYE_INNER = 362
LEFT_EYE_UPPER = 386
LEFT_EYE_LOWER = 374
LEFT_EYE_EAR = (362, 385, 387, 263, 373, 380)
LEFT_EYE_CONTOUR = (263, 466, 388, 387, 386, 385, 384, 398, 362, 382, 381, 380, 374, 373, 390, 249)
RIGHT_IRIS_CENTER = 468
RIGHT_IRIS_RING = (469, 470, 471, 472)
LEFT_IRIS_CENTER = 473
LEFT_IRIS_RING = (474, 475, 476, 477)
RIGHT_BROW = (70, 63, 105, 66, 107)
LEFT_BROW = (300, 293, 334, 296, 336)
RIGHT_BROW_MID = 105
LEFT_BROW_MID = 334
MOUTH_RIGHT = 61
MOUTH_LEFT = 291
UPPER_LIP_INNER = 13
LOWER_LIP_INNER = 14
UPPER_LIP_OUTER = 0
LOWER_LIP_OUTER = 17
LIPS_OUTER = (61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146)
LIPS_INNER = (78, 191, 80, 81, 82, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95)
FOREHEAD_TOP = 10
FOREHEAD_CENTER = 151
GLABELLA = 9
NOSE_BRIDGE = 168
NOSE_TIP = 1
CHIN = 152
RIGHT_CHEEK = 205
LEFT_CHEEK = 425
RIGHT_FACE_EDGE = 234
LEFT_FACE_EDGE = 454
RIGHT_NOSE_ALA = 129
LEFT_NOSE_ALA = 358
FACE_OVAL = (10, 338, 297, 332, 284, 251, 389, 356, 454, 323, 361, 288, 397, 365, 379, 378, 400, 377,
152, 148, 176, 149, 150, 136, 172, 58, 132, 93, 234, 127, 162, 21, 54, 103, 67, 109)
class HandIdx:
WRIST = 0
THUMB_CMC = 1
THUMB_MCP = 2
THUMB_IP = 3
THUMB_TIP = 4
INDEX_MCP = 5
INDEX_PIP = 6
INDEX_DIP = 7
INDEX_TIP = 8
MIDDLE_MCP = 9
MIDDLE_PIP = 10
MIDDLE_DIP = 11
MIDDLE_TIP = 12
RING_MCP = 13
RING_PIP = 14
RING_DIP = 15
RING_TIP = 16
PINKY_MCP = 17
PINKY_PIP = 18
PINKY_DIP = 19
PINKY_TIP = 20
FINGERS = {
"thumb": (1, 2, 3, 4),
"index": (5, 6, 7, 8),
"middle": (9, 10, 11, 12),
"ring": (13, 14, 15, 16),
"pinky": (17, 18, 19, 20),
}
class PoseIdx:
NOSE = 0
LEFT_EYE = 1
RIGHT_EYE = 2
LEFT_EAR = 3
RIGHT_EAR = 4
LEFT_SHOULDER = 5
RIGHT_SHOULDER = 6
LEFT_ELBOW = 7
RIGHT_ELBOW = 8
LEFT_WRIST = 9
RIGHT_WRIST = 10
LEFT_HIP = 11
RIGHT_HIP = 12
LEFT_KNEE = 13
RIGHT_KNEE = 14
LEFT_ANKLE = 15
RIGHT_ANKLE = 16
LEFT_BIG_TOE = 17
LEFT_SMALL_TOE = 18
LEFT_HEEL = 19
RIGHT_BIG_TOE = 20
RIGHT_SMALL_TOE = 21
RIGHT_HEEL = 22
def _box_from_points(points: np.ndarray, width: int, height: int) -> tuple[int, int, int, int]:
x_min = int(max(0, math.floor(float(points[:, 0].min()))))
y_min = int(max(0, math.floor(float(points[:, 1].min()))))
x_max = int(min(width - 1, math.ceil(float(points[:, 0].max()))))
y_max = int(min(height - 1, math.ceil(float(points[:, 1].max()))))
return x_min, y_min, x_max, y_max
def _wrap_angle(angle: float) -> float:
if angle > 90.0:
return angle - 180.0
if angle < -90.0:
return angle + 180.0
return angle
def _head_pose_from_matrix(matrix: Any) -> tuple[float, float, float]:
transform = np.asarray(matrix, dtype=np.float64)
if transform.shape != (4, 4):
return 0.0, 0.0, 0.0
rotation = transform[:3, :3].copy()
for column in range(3):
norm = float(np.linalg.norm(rotation[:, column]))
if norm > 1e-9:
rotation[:, column] /= norm
horizontal = math.sqrt(rotation[0, 0] ** 2 + rotation[1, 0] ** 2)
if horizontal > 1e-6:
pitch = math.degrees(math.atan2(rotation[2, 1], rotation[2, 2]))
yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal))
roll = math.degrees(math.atan2(rotation[1, 0], rotation[0, 0]))
else:
pitch = math.degrees(math.atan2(-rotation[1, 2], rotation[1, 1]))
yaw = math.degrees(math.atan2(-rotation[2, 0], horizontal))
roll = 0.0
return _wrap_angle(yaw), _wrap_angle(pitch), _wrap_angle(roll)
@dataclass
class FaceData:
points: np.ndarray
blendshapes: dict[str, float]
yaw: float
pitch: float
roll: float
iris_diameter_px: float
mm_per_px: float
box: tuple[int, int, int, int]
width_fraction: float
def point(self, index: int) -> np.ndarray:
return self.points[index, :2]
def points_2d(self, indices: Iterable[int]) -> np.ndarray:
return self.points[list(indices), :2]
def blendshape(self, name: str) -> float:
return float(self.blendshapes.get(name, 0.0))
def face_width_px(self) -> float:
return float(np.linalg.norm(self.points[FaceIdx.RIGHT_FACE_EDGE, :2] - self.points[FaceIdx.LEFT_FACE_EDGE, :2]))
@dataclass
class HandData:
side: str
points: np.ndarray
score: float
box: tuple[int, int, int, int]
size_fraction: float
def point(self, index: int) -> np.ndarray:
return self.points[index, :2]
def palm_width_px(self) -> float:
return float(np.linalg.norm(self.points[HandIdx.INDEX_MCP, :2] - self.points[HandIdx.PINKY_MCP, :2]))
def hand_length_px(self) -> float:
return float(np.linalg.norm(self.points[HandIdx.WRIST, :2] - self.points[HandIdx.MIDDLE_MCP, :2]))
@dataclass
class PoseData:
points: np.ndarray
scores: np.ndarray
box: tuple[int, int, int, int]
height_fraction: float
def visible(self, index: int, threshold: float = POSE_KEYPOINT_THRESHOLD) -> bool:
return bool(index < self.scores.shape[0] and float(self.scores[index]) >= threshold)
def point(self, index: int) -> np.ndarray:
return self.points[index, :2]
class SessionTracker:
def __init__(self) -> None:
self._face_landmarker: Any = None
self._hand_landmarker: Any = None
self._last_timestamps: dict[str, int] = {"face": -1, "hands": -1}
self._lock = threading.Lock()
self._closed = False
@staticmethod
def _require_file(path: str, label: str) -> None:
if not Path(path).is_file():
raise FileNotFoundError(f"{label} model file not found at {path}")
def _create_face_landmarker(self) -> Any:
from mediapipe.tasks import python as mp_python
from mediapipe.tasks.python import vision
self._require_file(SETTINGS.face_task_path, "Face landmarker")
options = vision.FaceLandmarkerOptions(
base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.face_task_path),
running_mode=vision.RunningMode.VIDEO,
num_faces=1,
min_face_detection_confidence=0.5,
min_face_presence_confidence=0.5,
min_tracking_confidence=0.5,
output_face_blendshapes=True,
output_facial_transformation_matrixes=True,
)
return vision.FaceLandmarker.create_from_options(options)
def _create_hand_landmarker(self) -> Any:
from mediapipe.tasks import python as mp_python
from mediapipe.tasks.python import vision
self._require_file(SETTINGS.hand_task_path, "Hand landmarker")
options = vision.HandLandmarkerOptions(
base_options=mp_python.BaseOptions(model_asset_path=SETTINGS.hand_task_path),
running_mode=vision.RunningMode.VIDEO,
num_hands=2,
min_hand_detection_confidence=0.5,
min_hand_presence_confidence=0.5,
min_tracking_confidence=0.5,
)
return vision.HandLandmarker.create_from_options(options)
def _next_timestamp(self, tracker_name: str, timestamp_ms: float) -> int:
last = self._last_timestamps[tracker_name]
timestamp = int(timestamp_ms)
if timestamp <= last:
timestamp = last + 1
self._last_timestamps[tracker_name] = timestamp
return timestamp
def process(
self,
image_bgr: np.ndarray,
timestamp_ms: float,
trackers: set[str],
) -> tuple[FaceData | None, dict[str, HandData], PoseData | None]:
face: FaceData | None = None
hands: dict[str, HandData] = {}
pose: PoseData | None = None
height, width = image_bgr.shape[:2]
with self._lock:
if self._closed:
return face, hands, pose
if "face" in trackers or "hands" in trackers:
import mediapipe as mp
rgb = np.ascontiguousarray(image_bgr[:, :, ::-1])
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
if "face" in trackers:
if self._face_landmarker is None:
self._face_landmarker = self._create_face_landmarker()
result = self._face_landmarker.detect_for_video(mp_image, self._next_timestamp("face", timestamp_ms))
face = self._convert_face(result, width, height)
if "hands" in trackers:
if self._hand_landmarker is None:
self._hand_landmarker = self._create_hand_landmarker()
result = self._hand_landmarker.detect_for_video(mp_image, self._next_timestamp("hands", timestamp_ms))
hands = self._convert_hands(result, width, height)
if "pose" in trackers:
pose = REGISTRY.detect_pose(image_bgr)
return face, hands, pose
@staticmethod
def _convert_face(result: Any, width: int, height: int) -> FaceData | None:
if not result.face_landmarks:
return None
landmarks = result.face_landmarks[0]
if len(landmarks) < 478:
return None
points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32)
blendshapes: dict[str, float] = {}
if result.face_blendshapes:
blendshapes = {category.category_name: float(category.score) for category in result.face_blendshapes[0]}
yaw, pitch, roll = 0.0, 0.0, 0.0
if result.facial_transformation_matrixes:
yaw, pitch, roll = _head_pose_from_matrix(result.facial_transformation_matrixes[0])
iris_diameters = []
for ring in (FaceIdx.RIGHT_IRIS_RING, FaceIdx.LEFT_IRIS_RING):
ring_points = points[list(ring), :2]
distances = [
float(np.linalg.norm(ring_points[i] - ring_points[j]))
for i in range(len(ring_points))
for j in range(i + 1, len(ring_points))
]
iris_diameters.append(max(distances))
iris_diameter = float(np.mean(iris_diameters))
mm_per_px = IRIS_DIAMETER_MM / iris_diameter if iris_diameter > 1.0 else 0.0
box = _box_from_points(points, width, height)
width_fraction = float((points[:, 0].max() - points[:, 0].min()) / max(1, width))
return FaceData(
points=points,
blendshapes=blendshapes,
yaw=yaw,
pitch=pitch,
roll=roll,
iris_diameter_px=iris_diameter,
mm_per_px=mm_per_px,
box=box,
width_fraction=width_fraction,
)
@staticmethod
def _convert_hands(result: Any, width: int, height: int) -> dict[str, HandData]:
hands: dict[str, HandData] = {}
if not result.hand_landmarks:
return hands
for landmarks, handedness in zip(result.hand_landmarks, result.handedness):
if not handedness:
continue
label = handedness[0].category_name.strip().lower()
score = float(handedness[0].score)
if SETTINGS.hand_label_swap:
side = "left" if label == "right" else "right"
else:
side = "right" if label == "right" else "left"
points = np.array([[lm.x * width, lm.y * height, lm.z * width] for lm in landmarks], dtype=np.float32)
box = _box_from_points(points, width, height)
size_fraction = float(max(box[2] - box[0], box[3] - box[1]) / max(1, width))
if side in hands and hands[side].score >= score:
continue
hands[side] = HandData(side=side, points=points, score=score, box=box, size_fraction=size_fraction)
return hands
def close(self) -> None:
with self._lock:
self._closed = True
for landmarker in (self._face_landmarker, self._hand_landmarker):
if landmarker is not None:
try:
landmarker.close()
except Exception:
logger.warning("Failed to close landmarker", exc_info=True)
self._face_landmarker = None
self._hand_landmarker = None
class ModelRegistry:
def __init__(self) -> None:
self.status: dict[str, str] = {"pose": "not_loaded", "qwen": "not_loaded"}
self.errors: dict[str, str] = {}
self._pose_model: Any = None
self._pose_init_lock = threading.Lock()
self._pose_run_lock = threading.Lock()
self._qwen_model: Any = None
self._qwen_processor: Any = None
self._qwen_load_lock = threading.Lock()
self._qwen_run_lock = threading.Lock()
self._qwen_finished = threading.Event()
self._qwen_started = False
self._token_cache: dict[str, int] = {}
def warm_up(self) -> None:
try:
self.pose_model()
self.detect_pose(np.zeros((480, 640, 3), dtype=np.uint8))
except Exception as error:
self.status["pose"] = "failed"
self.errors["pose"] = str(error)
logger.exception("Pose model failed to load")
if SETTINGS.load_qwen_on_startup:
self._qwen_started = True
self.load_qwen()
def pose_model(self) -> Any:
with self._pose_init_lock:
if self._pose_model is None:
self.status["pose"] = "loading"
from rtmlib import Wholebody
device = SETTINGS.pose_device
if device == "cuda":
try:
import onnxruntime
if "CUDAExecutionProvider" not in onnxruntime.get_available_providers():
logger.warning("CUDA provider unavailable for onnxruntime, pose runs on CPU")
device = "cpu"
except Exception:
device = "cpu"
self._pose_model = Wholebody(
to_openpose=False,
mode=SETTINGS.pose_mode,
backend="onnxruntime",
device=device,
)
self.status["pose"] = "ready"
logger.info("Pose model ready on %s", device)
return self._pose_model
def detect_pose(self, image_bgr: np.ndarray) -> PoseData | None:
model = self.pose_model()
with self._pose_run_lock:
keypoints, scores = model(image_bgr)
if keypoints is None or scores is None:
return None
keypoints = np.asarray(keypoints, dtype=np.float32)
scores = np.asarray(scores, dtype=np.float32)
if keypoints.size == 0:
return None
if keypoints.ndim == 2:
keypoints = keypoints[None, ...]
scores = scores[None, ...]
height, width = image_bgr.shape[:2]
best_pose: PoseData | None = None
best_area = 0.0
for person_points, person_scores in zip(keypoints, scores):
if float(np.mean(person_scores[:17])) < 0.2:
continue
body_points = person_points[:23]
visible_points = body_points[person_scores[:23] >= 0.3]
if visible_points.shape[0] < 4:
continue
box = _box_from_points(visible_points, width, height)
area = float((box[2] - box[0]) * (box[3] - box[1]))
if area > best_area:
best_area = area
best_pose = PoseData(
points=person_points.astype(np.float32),
scores=person_scores.astype(np.float32),
box=box,
height_fraction=float((box[3] - box[1]) / max(1, height)),
)
return best_pose
def load_qwen(self) -> None:
with self._qwen_load_lock:
if self._qwen_model is not None or self.status["qwen"] == "failed":
self._qwen_finished.set()
return
self.status["qwen"] = "loading"
try:
self._load_qwen_weights()
self.status["qwen"] = "ready"
logger.info("Qwen model ready: %s", SETTINGS.qwen_model_id)
except Exception as error:
self.status["qwen"] = "failed"
self.errors["qwen"] = str(error)
logger.exception("Qwen model failed to load")
finally:
self._qwen_finished.set()
def _load_qwen_weights(self) -> None:
import torch
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
token = os.getenv("HF_TOKEN") or None
model_kwargs: dict[str, Any] = {"device_map": "auto", "low_cpu_mem_usage": True, "token": token}
if torch.cuda.is_available():
if SETTINGS.qwen_precision == "4bit":
from transformers import BitsAndBytesConfig
model_kwargs["quantization_config"] = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_compute_dtype=torch.float16,
bnb_4bit_use_double_quant=True,
)
model_kwargs["dtype"] = torch.float16
elif SETTINGS.qwen_precision == "bfloat16":
model_kwargs["dtype"] = torch.bfloat16
else:
model_kwargs["dtype"] = torch.float16
else:
logger.warning("CUDA not available, Qwen will run on CPU and be very slow")
model_kwargs["dtype"] = torch.float32
processor = AutoProcessor.from_pretrained(SETTINGS.qwen_model_id, token=token)
try:
model = Qwen3VLForConditionalGeneration.from_pretrained(
SETTINGS.qwen_model_id, attn_implementation="sdpa", **model_kwargs
)
except (ValueError, ImportError, TypeError):
logger.warning("SDPA attention unavailable, loading with default attention", exc_info=True)
model = Qwen3VLForConditionalGeneration.from_pretrained(SETTINGS.qwen_model_id, **model_kwargs)
model.eval()
self._qwen_processor = processor
self._qwen_model = model
def wait_for_qwen(self, timeout_seconds: float) -> bool:
if self.status["qwen"] == "ready":
return True
if self.status["qwen"] == "failed":
return False
if not self._qwen_started:
self._qwen_started = True
threading.Thread(target=self.load_qwen, name="qwen-loader", daemon=True).start()
self._qwen_finished.wait(timeout_seconds)
return self.status["qwen"] == "ready"
def _prepare_inputs(self, messages: list[dict[str, Any]]) -> Any:
inputs = self._qwen_processor.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
return_dict=True,
return_tensors="pt",
)
inputs = inputs.to(self._qwen_model.device)
if "pixel_values" in inputs:
inputs["pixel_values"] = inputs["pixel_values"].to(self._qwen_model.dtype)
return inputs
def qwen_generate(self, messages: list[dict[str, Any]], max_new_tokens: int | None = None) -> str:
if self._qwen_model is None:
raise RuntimeError("Qwen model is not loaded")
import torch
with self._qwen_run_lock:
inputs = self._prepare_inputs(messages)
with torch.inference_mode():
output_ids = self._qwen_model.generate(
**inputs,
max_new_tokens=max_new_tokens or SETTINGS.qwen_max_new_tokens,
do_sample=False,
repetition_penalty=1.05,
)
prompt_length = inputs["input_ids"].shape[1]
generated = output_ids[:, prompt_length:]
text = self._qwen_processor.batch_decode(
generated, skip_special_tokens=True, clean_up_tokenization_spaces=False
)[0]
return text.strip()
def _letter_token_id(self, letter: str) -> int:
if letter not in self._token_cache:
token_ids = self._qwen_processor.tokenizer.encode(letter, add_special_tokens=False)
if not token_ids:
raise ValueError(f"Cannot tokenize option letter {letter}")
self._token_cache[letter] = int(token_ids[0])
return self._token_cache[letter]
def qwen_option_probabilities(self, messages: list[dict[str, Any]], letters: list[str]) -> list[float]:
if self._qwen_model is None:
raise RuntimeError("Qwen model is not loaded")
import torch
with self._qwen_run_lock:
inputs = self._prepare_inputs(messages)
with torch.inference_mode():
outputs = self._qwen_model(**inputs)
logits = outputs.logits[0, -1, :].float()
token_ids = [self._letter_token_id(letter) for letter in letters]
selected = logits[token_ids]
probabilities = torch.softmax(selected, dim=0).cpu().tolist()
return [float(value) for value in probabilities]
def gpu_info(self) -> dict[str, Any]:
try:
import torch
if not torch.cuda.is_available():
return {"available": False}
properties = torch.cuda.get_device_properties(0)
return {
"available": True,
"name": properties.name,
"total_memory_gb": round(properties.total_memory / 1e9, 2),
"allocated_memory_gb": round(torch.cuda.memory_allocated(0) / 1e9, 2),
}
except Exception:
return {"available": False}
REGISTRY = ModelRegistry()