| from __future__ import annotations |
|
|
| from functools import lru_cache |
| from pathlib import Path |
|
|
| import cv2 |
| import numpy as np |
| from PIL import Image, ImageDraw, ImageFont |
|
|
| from canvas import PALETTE |
| from gestures import LABELS, Gesture, GestureState |
| from hand import HAND_CONNECTIONS, INDEX_TIP, Hand |
|
|
| _FONT_CANDIDATES = ( |
| "/System/Library/Fonts/Supplemental/Arial.ttf", |
| "/System/Library/Fonts/Supplemental/Arial Unicode.ttf", |
| "/System/Library/Fonts/HelveticaNeue.ttc", |
| "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", |
| "C:/Windows/Fonts/arial.ttf", |
| ) |
|
|
| _ACCENT = { |
| Gesture.DRAW: (90, 230, 90), |
| Gesture.MOVE: (255, 190, 60), |
| Gesture.GRAB: (60, 220, 255), |
| Gesture.OPEN_PALM: (230, 230, 230), |
| Gesture.ZOOM: (255, 110, 200), |
| Gesture.OK: (120, 120, 255), |
| Gesture.MIDDLE: (80, 80, 240), |
| Gesture.PINKY: (200, 160, 255), |
| Gesture.RING: (255, 210, 120), |
| Gesture.NONE: (150, 150, 150), |
| } |
|
|
| HELP_LINES = ( |
| "GESTURES", |
| " One index finger - draw", |
| " Index and middle - move cursor", |
| " Fist (hold) - grab, move to rotate in 3D", |
| " Two fists - scale", |
| " Open palm - release", |
| " OK (hold) - clear canvas", |
| " Pinky (hold) - undo last stroke", |
| " Middle finger - :((", |
| " Two palms waving - quit", |
| "", |
| "KEYS", |
| " S save C clear U undo", |
| " 1..6 color [ ] size R reset view", |
| " SPACE 2D/3D K skeleton D debug", |
| " H help Q quit", |
| ) |
|
|
| HELP_LINES_3D = ( |
| "GESTURES", |
| " One index finger - draw in depth", |
| " Index and middle - move cursor", |
| " Fist - rotate the drawing", |
| " Two fists - scale", |
| " Two ring fingers - depth colors", |
| " Open palm (hold) - reset view", |
| " OK (hold) - clear canvas", |
| " Pinky (hold) - undo last stroke", |
| " Middle finger - :((", |
| " Two palms waving - quit", |
| "", |
| "KEYS", |
| " S save C clear U undo", |
| " 1..6 solid color G depth gradient [ ] size", |
| " R reset view SPACE depth colors K skeleton D debug", |
| " H help Q quit", |
| ) |
|
|
|
|
| @lru_cache(maxsize=1) |
| def _font_path() -> str | None: |
| """Find available font.""" |
| for p in _FONT_CANDIDATES: |
| if Path(p).exists(): |
| return p |
| return None |
|
|
|
|
| @lru_cache(maxsize=8) |
| def _font(size: int) -> ImageFont.FreeTypeFont | ImageFont.ImageFont: |
| """Load font at size.""" |
| path = _font_path() |
| if path is None: |
| return ImageFont.load_default() |
| return ImageFont.truetype(path, size) |
|
|
|
|
| @lru_cache(maxsize=256) |
| def _render_text(text: str, size: int, color: tuple[int, int, int]) -> tuple[np.ndarray, np.ndarray]: |
| """Rasterize and cache text.""" |
| font = _font(size) |
| dummy = ImageDraw.Draw(Image.new("L", (1, 1))) |
| box = dummy.textbbox((0, 0), text, font=font) |
| w = max(1, box[2] - box[0] + 4) |
| h = max(1, box[3] - box[1] + 4) |
| img = Image.new("L", (w, h), 0) |
| ImageDraw.Draw(img).text((2 - box[0], 2 - box[1]), text, font=font, fill=255) |
| alpha = np.array(img, dtype=np.float32) / 255.0 |
| rgb = np.zeros((h, w, 3), dtype=np.float32) |
| rgb[:, :] = color |
| return rgb, alpha |
|
|
|
|
| def put_text( |
| frame: np.ndarray, |
| text: str, |
| org: tuple[int, int], |
| size: int = 18, |
| color: tuple[int, int, int] = (240, 240, 240), |
| ) -> int: |
| """Draw text string.""" |
| if not text: |
| return size + 6 |
| rgb, alpha = _render_text(text, size, color) |
| h, w = alpha.shape |
| x, y = org |
| fh, fw = frame.shape[:2] |
| if x >= fw or y >= fh: |
| return h |
| x0, y0 = max(0, x), max(0, y) |
| x1, y1 = min(fw, x + w), min(fh, y + h) |
| if x1 <= x0 or y1 <= y0: |
| return h |
| sub_a = alpha[y0 - y : y1 - y, x0 - x : x1 - x][:, :, None] |
| sub_c = rgb[y0 - y : y1 - y, x0 - x : x1 - x] |
| roi = frame[y0:y1, x0:x1].astype(np.float32) |
| frame[y0:y1, x0:x1] = (roi * (1 - sub_a) + sub_c * sub_a).astype(np.uint8) |
| return h |
|
|
|
|
| def panel(frame: np.ndarray, x: int, y: int, w: int, h: int, alpha: float = 0.55) -> None: |
| """Dark translucent backing.""" |
| fh, fw = frame.shape[:2] |
| x0, y0 = max(0, x), max(0, y) |
| x1, y1 = min(fw, x + w), min(fh, y + h) |
| if x1 <= x0 or y1 <= y0: |
| return |
| roi = frame[y0:y1, x0:x1] |
| dark = np.zeros_like(roi) |
| cv2.addWeighted(dark, alpha, roi, 1 - alpha, 0, roi) |
|
|
|
|
| def draw_hand(frame: np.ndarray, hand: Hand, state: GestureState, kpt_conf: float = 0.3) -> None: |
| """Draw hand skeleton.""" |
| kp = hand.keypoints |
| ok = hand.scores >= kpt_conf |
| color = _ACCENT.get(state.gesture, (150, 150, 150)) |
| for a, b in HAND_CONNECTIONS: |
| if ok[a] and ok[b]: |
| cv2.line(frame, tuple(kp[a].astype(int)), tuple(kp[b].astype(int)), |
| (60, 60, 60), 3, cv2.LINE_AA) |
| cv2.line(frame, tuple(kp[a].astype(int)), tuple(kp[b].astype(int)), |
| color, 1, cv2.LINE_AA) |
| for i in range(21): |
| if ok[i]: |
| cv2.circle(frame, tuple(kp[i].astype(int)), 3, color, -1, cv2.LINE_AA) |
| if state.gesture == Gesture.DRAW: |
| cv2.circle(frame, tuple(kp[INDEX_TIP].astype(int)), 9, (90, 230, 90), 2, cv2.LINE_AA) |
| elif state.gesture == Gesture.GRAB: |
| cv2.circle(frame, tuple(hand.center.astype(int)), 16, (60, 220, 255), 2, cv2.LINE_AA) |
|
|
|
|
| _POSE_LIMB = (255, 210, 90) |
| _POSE_FACE = (190, 240, 170) |
| _POSE_JOINT = (255, 255, 255) |
| _POSE_WEAK = (110, 90, 90) |
|
|
|
|
| def draw_pose(frame: np.ndarray, pose, threshold: float = 0.35, |
| labels: bool = False) -> None: |
| """Draw one body skeleton with its keypoints.""" |
| from pose import FACE, KEYPOINT_NAMES, SKELETON |
|
|
| kp = pose.keypoints |
| ok = pose.scores >= threshold |
| |
| |
| k = max(frame.shape[1] / 1280.0, 0.35) |
| thin = max(int(round(2 * k)), 1) |
| halo = thin + max(int(round(3 * k)), 2) |
|
|
| for a, b in SKELETON: |
| pa, pb = tuple(kp[a].astype(int)), tuple(kp[b].astype(int)) |
| if ok[a] and ok[b]: |
| color = _POSE_FACE if (a in FACE and b in FACE) else _POSE_LIMB |
| cv2.line(frame, pa, pb, (25, 25, 30), halo, cv2.LINE_AA) |
| cv2.line(frame, pa, pb, color, thin, cv2.LINE_AA) |
| elif ok[a] or ok[b]: |
| cv2.line(frame, pa, pb, _POSE_WEAK, max(thin - 1, 1), cv2.LINE_AA) |
|
|
| for i in range(len(kp)): |
| p = tuple(kp[i].astype(int)) |
| if not ok[i]: |
| cv2.circle(frame, p, max(int(round(2 * k)), 1), _POSE_WEAK, -1, cv2.LINE_AA) |
| continue |
| radius = max(int(round((3 if i in FACE else 5) * k)), 2) |
| cv2.circle(frame, p, radius + max(int(round(2 * k)), 1), (25, 25, 30), -1, cv2.LINE_AA) |
| cv2.circle(frame, p, radius, _POSE_JOINT, -1, cv2.LINE_AA) |
| if labels and i not in FACE: |
| put_text(frame, KEYPOINT_NAMES[i], (p[0] + 8, p[1] - 8), 13, (200, 230, 255)) |
|
|
|
|
| def draw_cursor(frame: np.ndarray, point: np.ndarray, gesture: Gesture, |
| color: tuple[int, int, int], thickness: int) -> None: |
| """Draw pen cursor.""" |
| p = tuple(np.asarray(point, dtype=int)) |
| if gesture == Gesture.DRAW: |
| cv2.circle(frame, p, max(4, thickness), color, -1, cv2.LINE_AA) |
| cv2.circle(frame, p, max(4, thickness) + 5, (255, 255, 255), 1, cv2.LINE_AA) |
| elif gesture == Gesture.MOVE: |
| cv2.circle(frame, p, max(6, thickness + 4), (255, 190, 60), 2, cv2.LINE_AA) |
| cv2.drawMarker(frame, p, (255, 190, 60), cv2.MARKER_CROSS, 14, 1, cv2.LINE_AA) |
|
|
|
|
| def draw_hud( |
| frame: np.ndarray, |
| *, |
| gesture: Gesture, |
| hands_info: list | None = None, |
| mode_3d: bool, |
| fps: float, |
| device: str, |
| color: tuple[int, int, int], |
| thickness: int, |
| strokes: int, |
| clear_progress: float = 0.0, |
| toast: str = "", |
| mode_label: str | None = None, |
| hint: str = "Finger draw Fist 3D Palm release", |
| ) -> None: |
| """Draw status overlay.""" |
| h, w = frame.shape[:2] |
|
|
| panel(frame, 0, 0, w, 78, 0.5) |
| mode = mode_label or ("3D VIEW" if mode_3d else "DRAW") |
| put_text(frame, mode, (16, 10), 20, (255, 255, 255)) |
| put_text(frame, f"Gesture: {LABELS[gesture]}", (16, 40), 17, _ACCENT.get(gesture, (200, 200, 200))) |
|
|
| hands_txt = ", ".join(f"{h}" for h, _ in (hands_info or [])) or "no hands" |
| right = f"{fps:.0f} FPS {device.upper()} strokes {strokes} {hands_txt}" |
| rw = _render_text(right, 16, (200, 200, 200))[0].shape[1] |
| put_text(frame, right, (w - rw - 16, 14), 16, (200, 200, 200)) |
| put_text(frame, "H help", (w - rw - 16, 40), 15, (140, 140, 140)) |
|
|
| panel(frame, 0, h - 46, w, 46, 0.5) |
| x = 16 |
| for c in PALETTE: |
| cv2.rectangle(frame, (x, h - 34), (x + 24, h - 12), c, -1, cv2.LINE_AA) |
| if c == color: |
| cv2.rectangle(frame, (x - 3, h - 37), (x + 27, h - 9), (255, 255, 255), 2, cv2.LINE_AA) |
| x += 34 |
| put_text(frame, f"size {thickness}", (x + 10, h - 34), 16, (220, 220, 220)) |
| hw = _render_text(hint, 15, (170, 170, 170))[0].shape[1] |
| put_text(frame, hint, (w - hw - 16, h - 32), 15, (170, 170, 170)) |
| cv2.circle(frame, (x + 130 + thickness, h - 23), max(1, thickness // 2), color, -1, cv2.LINE_AA) |
|
|
| if clear_progress > 0: |
| bw = int(260 * min(1.0, clear_progress)) |
| cv2.rectangle(frame, (w // 2 - 130, h - 78), (w // 2 + 130, h - 62), (40, 40, 40), -1) |
| cv2.rectangle(frame, (w // 2 - 130, h - 78), (w // 2 - 130 + bw, h - 62), (80, 80, 255), -1) |
| put_text(frame, "grabbing", (w // 2 - 30, h - 104), 16, (200, 240, 255)) |
|
|
| if toast: |
| tw = _render_text(toast, 18, (255, 255, 255))[0].shape[1] |
| panel(frame, w // 2 - tw // 2 - 14, 92, tw + 28, 36, 0.65) |
| put_text(frame, toast, (w // 2 - tw // 2, 100), 18, (255, 255, 255)) |
|
|
|
|
| def draw_debug(frame: np.ndarray, hands, states) -> None: |
| """Draw classifier debug panel.""" |
| names = ("thumb", "index", "middl", "ring ", "pinky") |
| y = 96 |
| for hand, st in zip(hands, states): |
| panel(frame, 12, y, 300, 24 + 16 * 7, 0.66) |
| yy = y + 16 |
| cv2.putText(frame, f"{st.handedness[:4]} conf {st.confidence:.2f} " |
| f"{'stable' if st.stable else 'JITTER'}", |
| (20, yy), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv2.LINE_AA) |
| yy += 16 |
| for i, name in enumerate(names): |
| ratio = st.ratios[i] if i < len(st.ratios) else 1.0 |
| curl = st.curls[i] if i < len(st.curls) else 0.0 |
| up = st.fingers[i] |
| col = (110, 240, 110) if up else (140, 140, 220) |
| bar = "#" * int(min(max(ratio - 0.7, 0) * 12, 10)) |
| cv2.putText(frame, f"{name} x{ratio:4.2f} {curl:3.0f}d " |
| f"{'UP ' if up else 'down'} {bar}", |
| (20, yy), cv2.FONT_HERSHEY_SIMPLEX, 0.4, col, 1, cv2.LINE_AA) |
| yy += 16 |
| closed = st.pinch_distance < 0.62 |
| cv2.putText(frame, f"pinch {st.pinch_distance:5.2f} " |
| f"{'CLOSED' if closed else 'open'} lowconf " |
| f"{int((hand.scores < 0.25).sum())}/21", |
| (20, yy), cv2.FONT_HERSHEY_SIMPLEX, 0.4, |
| (110, 240, 110) if closed else (180, 180, 180), 1, cv2.LINE_AA) |
| y += 24 + 16 * 7 + 8 |
|
|
|
|
| def draw_help(frame: np.ndarray, lines: tuple[str, ...] = HELP_LINES) -> None: |
| """Draw help overlay.""" |
| h, w = frame.shape[:2] |
| bw, bh = min(w - 60, 720), 20 + len(lines) * 26 |
| x, y = (w - bw) // 2, (h - bh) // 2 |
| panel(frame, x, y, bw, bh, 0.78) |
| cv2.rectangle(frame, (x, y), (x + bw, y + bh), (90, 90, 90), 1, cv2.LINE_AA) |
| cy = y + 12 |
| for line in lines: |
| bold = line and not line.startswith(" ") |
| put_text(frame, line, (x + 22, cy), 17 if bold else 16, |
| (255, 255, 255) if bold else (215, 215, 215)) |
| cy += 26 |
|
|
