import av import cv2 import math import numpy as np import mediapipe as mp import streamlit as st from streamlit_webrtc import webrtc_streamer, VideoProcessorBase st.set_page_config(page_title="NeuroNav-CV Demo", layout="wide") st.title("NeuroNav-CV: Markerless Head Tracking Demo") st.write( "Early proof-of-concept using a standard webcam to detect facial landmarks, " "generate a live facial mesh, and estimate head pose without physical markers." ) mp_face_mesh = mp.solutions.face_mesh mp_drawing = mp.solutions.drawing_utils mp_styles = mp.solutions.drawing_styles FACE_3D_MODEL_POINTS = np.array([ (0.0, 0.0, 0.0), (0.0, -63.6, -12.5), (-43.3, 32.7, -26.0), (43.3, 32.7, -26.0), (-28.9, -28.9, -24.1), (28.9, -28.9, -24.1), ], dtype=np.float64) LANDMARK_IDS = { "nose": 1, "chin": 152, "left_eye": 33, "right_eye": 263, "left_mouth": 61, "right_mouth": 291, } def rotation_matrix_to_euler_angles(rotation_matrix): sy = math.sqrt(rotation_matrix[0, 0] ** 2 + rotation_matrix[1, 0] ** 2) singular = sy < 1e-6 if not singular: pitch = math.atan2(rotation_matrix[2, 1], rotation_matrix[2, 2]) yaw = math.atan2(-rotation_matrix[2, 0], sy) roll = math.atan2(rotation_matrix[1, 0], rotation_matrix[0, 0]) else: pitch = math.atan2(-rotation_matrix[1, 2], rotation_matrix[1, 1]) yaw = math.atan2(-rotation_matrix[2, 0], sy) roll = 0 return np.degrees([pitch, yaw, roll]) class NeuroNavVideoProcessor(VideoProcessorBase): def __init__(self): self.face_mesh = mp_face_mesh.FaceMesh( max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.5, min_tracking_confidence=0.5, ) def recv(self, frame): img = frame.to_ndarray(format="bgr24") img = cv2.flip(img, 1) h, w = img.shape[:2] rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) results = self.face_mesh.process(rgb) if results.multi_face_landmarks: face_landmarks = results.multi_face_landmarks[0] mp_drawing.draw_landmarks( image=img, landmark_list=face_landmarks, connections=mp_face_mesh.FACEMESH_TESSELATION, landmark_drawing_spec=None, connection_drawing_spec=mp_styles.get_default_face_mesh_tesselation_style(), ) mp_drawing.draw_landmarks( image=img, landmark_list=face_landmarks, connections=mp_face_mesh.FACEMESH_CONTOURS, landmark_drawing_spec=None, connection_drawing_spec=mp_styles.get_default_face_mesh_contours_style(), ) image_points = [] for key in ["nose", "chin", "left_eye", "right_eye", "left_mouth", "right_mouth"]: lm = face_landmarks.landmark[LANDMARK_IDS[key]] x, y = int(lm.x * w), int(lm.y * h) image_points.append((x, y)) cv2.circle(img, (x, y), 5, (0, 255, 255), -1) image_points = np.array(image_points, dtype=np.float64) focal_length = w camera_matrix = np.array([ [focal_length, 0, w / 2], [0, focal_length, h / 2], [0, 0, 1] ], dtype=np.float64) dist_coeffs = np.zeros((4, 1)) success_pnp, rotation_vector, translation_vector = cv2.solvePnP( FACE_3D_MODEL_POINTS, image_points, camera_matrix, dist_coeffs, flags=cv2.SOLVEPNP_ITERATIVE ) if success_pnp: rotation_matrix, _ = cv2.Rodrigues(rotation_vector) pitch, yaw, roll = rotation_matrix_to_euler_angles(rotation_matrix) nose_2d = tuple(image_points[0].astype(int)) nose_projection, _ = cv2.projectPoints( np.array([(0.0, 0.0, 100.0)], dtype=np.float64), rotation_vector, translation_vector, camera_matrix, dist_coeffs ) nose_direction = tuple(nose_projection[0][0].astype(int)) cv2.line(img, nose_2d, nose_direction, (255, 0, 0), 3) cv2.putText(img, "TRACKING: LOCKED", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 255, 0), 2) cv2.putText(img, f"Pitch: {pitch:.1f}", (20, 75), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2) cv2.putText(img, f"Yaw: {yaw:.1f}", (20, 110), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2) cv2.putText(img, f"Roll: {roll:.1f}", (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 255, 0), 2) cv2.putText(img, f"X: {translation_vector[0][0]:.1f}", (20, 190), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2) cv2.putText(img, f"Y: {translation_vector[1][0]:.1f}", (20, 220), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2) cv2.putText(img, f"Z: {translation_vector[2][0]:.1f}", (20, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.65, (255, 255, 0), 2) else: cv2.putText(img, "TRACKING: SEARCHING", (20, 35), cv2.FONT_HERSHEY_SIMPLEX, 0.9, (0, 0, 255), 2) return av.VideoFrame.from_ndarray(img, format="bgr24") st.info( "Click START below and allow webcam access. Move your head left/right/up/down. " "This is not yet MRI/DICOM registration; it is the first markerless tracking proof-of-concept." ) webrtc_streamer( key="neuronav-cv-demo", video_processor_factory=NeuroNavVideoProcessor, media_stream_constraints={"video": True, "audio": False}, )