| from __future__ import annotations |
|
|
| from collections import Counter, deque |
| from dataclasses import dataclass, field |
| from enum import Enum |
|
|
| import numpy as np |
|
|
| from hand import ( |
| INDEX_MCP, |
| INDEX_PIP, |
| INDEX_TIP, |
| PINKY_MCP, |
| THUMB_TIP, |
| WRIST, |
| Hand, |
| ) |
|
|
|
|
| class Gesture(str, Enum): |
| NONE = "none" |
| DRAW = "draw" |
| MOVE = "move" |
| GRAB = "grab" |
| OPEN_PALM = "palm" |
| ZOOM = "zoom" |
| OK = "ok" |
| MIDDLE = "middle" |
| PINKY = "pinky" |
| RING = "ring" |
|
|
|
|
| LABELS: dict[Gesture, str] = { |
| Gesture.NONE: "none", |
| Gesture.DRAW: "Draw", |
| Gesture.MOVE: "Move", |
| Gesture.GRAB: "Grab", |
| Gesture.OPEN_PALM: "Palm", |
| Gesture.ZOOM: "Zoom", |
| Gesture.OK: "OK", |
| Gesture.MIDDLE: "Middle", |
| Gesture.PINKY: "Undo", |
| Gesture.RING: "Ring", |
| } |
|
|
| _CHAINS: tuple[tuple[int, int, int, int], ...] = ( |
| (1, 2, 3, 4), |
| (5, 6, 7, 8), |
| (9, 10, 11, 12), |
| (13, 14, 15, 16), |
| (17, 18, 19, 20), |
| ) |
|
|
| _E_LO = 0.96 |
| _FIST_MAX = 1.04 |
| _UP_ON = 0.72 |
| _UP_OFF = 0.60 |
| _UP_LOW_ON = 0.82 |
| _UP_LOW_OFF = 0.50 |
|
|
| _PINCH_ON = 0.42 |
| _PINCH_OFF = 0.62 |
|
|
| _MIN_KPT_CONF = 0.25 |
| _STRAIGHT_MARGIN = 32.0 |
|
|
|
|
| @dataclass |
| class GestureState: |
| """One hand classification result.""" |
|
|
| gesture: Gesture = Gesture.NONE |
| fingers: tuple[bool, bool, bool, bool, bool] = (False,) * 5 |
| curls: tuple[float, ...] = (0.0,) * 5 |
| ratios: tuple[float, ...] = (1.0,) * 5 |
| pinch_distance: float = 1.0 |
| cursor: np.ndarray = field(default_factory=lambda: np.zeros(2)) |
| pinch_point: np.ndarray = field(default_factory=lambda: np.zeros(2)) |
| confidence: float = 0.0 |
| handedness: str = "?" |
| stable: bool = False |
|
|
|
|
| def _angle(v1: np.ndarray, v2: np.ndarray) -> float: |
| """Angle between vectors.""" |
| n1, n2 = np.linalg.norm(v1), np.linalg.norm(v2) |
| if n1 < 1e-6 or n2 < 1e-6: |
| return 0.0 |
| c = float(np.clip(np.dot(v1, v2) / (n1 * n2), -1.0, 1.0)) |
| return float(np.degrees(np.arccos(c))) |
|
|
|
|
| def finger_curl(kp: np.ndarray, chain: tuple[int, int, int, int]) -> float: |
| """Finger bend in degrees.""" |
| mcp, pip, _dip, tip = chain |
| return _angle(kp[pip] - kp[mcp], kp[tip] - kp[pip]) |
|
|
|
|
| def handedness(kp: np.ndarray) -> str: |
| """Left or right hand.""" |
| v1 = kp[INDEX_MCP] - kp[WRIST] |
| v2 = kp[PINKY_MCP] - kp[WRIST] |
| z = v1[0] * v2[1] - v1[1] * v2[0] |
| return "right" if z > 0 else "left" |
|
|
|
|
| class GestureRecognizer: |
| """Single hand gesture classifier.""" |
|
|
| def __init__(self, vote_window: int = 5, vote_ratio: float = 0.6) -> None: |
| self.vote_window = vote_window |
| self.vote_ratio = vote_ratio |
| self._history: deque[Gesture] = deque(maxlen=vote_window) |
| self._stable = Gesture.NONE |
| self._extended = [False] * 5 |
| self._extendedness = [0.0] * 5 |
| self._fist = False |
| self._seen = False |
| self._pinching = False |
|
|
| def reset(self) -> None: |
| """Clear tracking state.""" |
| self._history.clear() |
| self._stable = Gesture.NONE |
| self._extended = [False] * 5 |
| self._extendedness = [0.0] * 5 |
| self._fist = False |
| self._seen = False |
| self._pinching = False |
|
|
| def _update_fingers(self, kp: np.ndarray, scores: np.ndarray) -> tuple[list[float], list[float]]: |
| """Decide each finger up.""" |
| curls, ratios = [], [] |
| for i, chain in enumerate(_CHAINS): |
| mcp, pip, _dip, tip = chain |
| curls.append(finger_curl(kp, chain)) |
| if i == 0: |
| ref, near = kp[PINKY_MCP], kp[mcp] |
| else: |
| ref, near = kp[WRIST], kp[pip] |
| d_far = float(np.linalg.norm(kp[tip] - ref)) |
| d_near = float(np.linalg.norm(near - ref)) |
| ratios.append(d_far / max(d_near, 1e-6)) |
|
|
| hi = max(ratios[1:]) |
| self._fist = hi < _FIST_MAX |
| span = max(hi - _E_LO, 1e-3) |
|
|
| for i, chain in enumerate(_CHAINS): |
| mcp, pip, _dip, tip = chain |
| e = float(np.clip((ratios[i] - _E_LO) / span, 0.0, 1.0)) |
| self._extendedness[i] = e |
| sure = min(float(scores[tip]), float(scores[mcp if i == 0 else pip])) |
|
|
| if not self._seen: |
| self._extended[i] = e > _UP_ON |
| continue |
| on, off = (_UP_LOW_ON, _UP_LOW_OFF) if sure < _MIN_KPT_CONF else (_UP_ON, _UP_OFF) |
| self._extended[i] = e > (off if self._extended[i] else on) |
| self._seen = True |
| return curls, ratios |
|
|
| def __call__(self, hand: Hand | None) -> GestureState: |
| """Classify one hand.""" |
| if hand is None: |
| self.reset() |
| return GestureState() |
|
|
| kp, scores = hand.keypoints, hand.scores |
| curls, ratios = self._update_fingers(kp, scores) |
|
|
| finger_len = float(np.linalg.norm(kp[INDEX_TIP] - kp[INDEX_PIP])) + \ |
| float(np.linalg.norm(kp[INDEX_PIP] - kp[INDEX_MCP])) |
| palm_w = float(np.linalg.norm(kp[PINKY_MCP] - kp[INDEX_MCP])) |
| ref = max(finger_len, palm_w * 0.85, 1e-3) |
| pinch_d = float(np.linalg.norm(kp[THUMB_TIP] - kp[INDEX_TIP]) / ref) |
| self._pinching = pinch_d < (_PINCH_OFF if self._pinching else _PINCH_ON) |
|
|
| raw = self._classify(tuple(self._extended), tuple(self._extendedness), |
| tuple(curls), self._fist, self._pinching) |
|
|
| self._history.append(raw) |
| counts = Counter(self._history) |
| winner, n = counts.most_common(1)[0] |
| stable = n >= max(2, int(self.vote_window * self.vote_ratio)) |
| if stable: |
| self._stable = winner |
|
|
| pinch_point = (kp[THUMB_TIP] + kp[INDEX_TIP]) * 0.5 |
| return GestureState( |
| gesture=self._stable, |
| fingers=tuple(self._extended), |
| curls=tuple(curls), |
| ratios=tuple(ratios), |
| pinch_distance=pinch_d, |
| cursor=kp[INDEX_TIP].copy(), |
| pinch_point=pinch_point, |
| confidence=hand.conf, |
| handedness=handedness(kp), |
| stable=stable, |
| ) |
|
|
| @staticmethod |
| def _classify(ext: tuple[bool, ...], e: tuple[float, ...], |
| curls: tuple[float, ...], fist: bool, pinching: bool) -> Gesture: |
| """Map fingers to gesture.""" |
| _thumb, index, middle, ring, pinky = ext |
| long_up = sum((index, middle, ring, pinky)) |
|
|
| if pinching and middle and ring and pinky and not index: |
| return Gesture.OK |
|
|
| if middle and not index and not ring and not pinky: |
| return Gesture.MIDDLE |
|
|
| if pinky and not index and not middle and not ring: |
| return Gesture.PINKY |
|
|
| if ring and not index and not middle and not pinky: |
| return Gesture.RING |
|
|
| if fist: |
| return Gesture.GRAB |
|
|
| if index and middle: |
| base = min(curls[1], curls[2]) |
| if (curls[3] < base + _STRAIGHT_MARGIN |
| and curls[4] < base + _STRAIGHT_MARGIN): |
| return Gesture.OPEN_PALM |
| return Gesture.MOVE |
|
|
| if index: |
| return Gesture.DRAW |
| if long_up <= 1: |
| return Gesture.GRAB |
| return Gesture.NONE |
|
|
