Download test_full_model.py from meghmodi/Indian-Sign-Language-KairoAI: direct link, hf CLI and curl.
- Browser
- Download file 21.4 kB
-
https://huggingface.co/meghmodi/Indian-Sign-Language-KairoAI/resolve/main/test_full_model.py
- Command line
-
hf download hf://meghmodi/Indian-Sign-Language-KairoAI/test_full_model.py
-
curl -L -o test_full_model.py https://huggingface.co/meghmodi/Indian-Sign-Language-KairoAI/resolve/main/test_full_model.py
21.4 kB
| """ | |
| ================================================================================ | |
| TEST ISL MODEL WITH MEDIAPIPE - WEBCAM REAL-TIME INFERENCE | |
| ================================================================================ | |
| This script tests the trained model (isl_model_full.tflite) using MediaPipe | |
| for hand landmark detection and orientation calculation. | |
| Features: | |
| - Real-time webcam hand detection | |
| - Extracts 130 features (126 landmarks + 4 orientation) | |
| - Runs TFLite inference | |
| - Displays prediction with confidence | |
| Controls: | |
| - Press 'q' to quit | |
| - Press 's' to save screenshot | |
| - Press 'c' to toggle confidence threshold | |
| Author: KairoAI | |
| ================================================================================ | |
| """ | |
| import cv2 | |
| import numpy as np | |
| import mediapipe as mp | |
| import tensorflow as tf | |
| import json | |
| import os | |
| from collections import deque | |
| # ============================================================================ | |
| # CONFIGURATION | |
| # ============================================================================ | |
| MODEL_PATH = "isl_model_full.tflite" | |
| LABELS_PATH = "labels_full.json" | |
| # Fallback paths if full model not found | |
| FALLBACK_MODEL = "isl_model.tflite" | |
| FALLBACK_LABELS = "labels.json" | |
| # Display settings | |
| CONFIDENCE_THRESHOLD = 0.5 | |
| SMOOTHING_WINDOW = 5 # Number of frames to average predictions | |
| SHOW_LANDMARKS = True | |
| SHOW_ORIENTATION = True | |
| # Colors (BGR) | |
| COLOR_PALM = (0, 255, 0) # Green for palm | |
| COLOR_BACK = (0, 165, 255) # Orange for back of hand | |
| COLOR_TEXT = (255, 255, 255) # White | |
| COLOR_BOX = (50, 50, 50) # Dark gray | |
| # Labels (fallback if json not found) | |
| DEFAULT_LABELS = [ | |
| 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', | |
| 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', | |
| 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2', '3', '4', | |
| '5', '6', '7', '8', '9' | |
| ] | |
| # ============================================================================ | |
| # HAND ORIENTATION CALCULATOR | |
| # ============================================================================ | |
| class HandOrientationCalculator: | |
| """ | |
| Calculate whether palm or back of hand is facing the camera. | |
| Uses the cross product of vectors on the hand plane. | |
| """ | |
| # Landmark indices | |
| WRIST = 0 | |
| INDEX_MCP = 5 | |
| PINKY_MCP = 17 | |
| MIDDLE_MCP = 9 | |
| def calculate_orientation(landmarks, handedness): | |
| """ | |
| Calculate hand orientation. | |
| Returns: (is_palm_facing: float, is_left_hand: float) | |
| - is_palm_facing: 1.0 = palm facing camera, 0.0 = back facing camera | |
| - is_left_hand: 1.0 = left hand, 0.0 = right hand | |
| """ | |
| if landmarks is None: | |
| return -1.0, -1.0 | |
| # Get key points | |
| wrist = np.array([ | |
| landmarks[HandOrientationCalculator.WRIST].x, | |
| landmarks[HandOrientationCalculator.WRIST].y, | |
| landmarks[HandOrientationCalculator.WRIST].z | |
| ]) | |
| index_mcp = np.array([ | |
| landmarks[HandOrientationCalculator.INDEX_MCP].x, | |
| landmarks[HandOrientationCalculator.INDEX_MCP].y, | |
| landmarks[HandOrientationCalculator.INDEX_MCP].z | |
| ]) | |
| pinky_mcp = np.array([ | |
| landmarks[HandOrientationCalculator.PINKY_MCP].x, | |
| landmarks[HandOrientationCalculator.PINKY_MCP].y, | |
| landmarks[HandOrientationCalculator.PINKY_MCP].z | |
| ]) | |
| # Calculate vectors on the palm plane | |
| v1 = index_mcp - wrist # Wrist to index | |
| v2 = pinky_mcp - wrist # Wrist to pinky | |
| # Cross product gives normal to palm plane | |
| normal = np.cross(v1, v2) | |
| # Z component of normal indicates palm orientation | |
| # Positive = palm facing camera, Negative = back facing | |
| z_component = normal[2] | |
| # Determine handedness | |
| is_left = 1.0 if handedness == "Left" else 0.0 | |
| # For left hand, the normal direction is reversed | |
| if is_left: | |
| z_component = -z_component | |
| # Convert to 0/1 (back/palm) | |
| is_palm_facing = 1.0 if z_component > 0 else 0.0 | |
| return is_palm_facing, is_left | |
| # ============================================================================ | |
| # LANDMARK PROCESSOR | |
| # ============================================================================ | |
| class LandmarkProcessor: | |
| """Process MediaPipe hand landmarks into model input features.""" | |
| def __init__(self, input_size=130): | |
| self.input_size = input_size | |
| self.has_orientation = input_size == 130 | |
| def normalize_landmarks(self, landmarks): | |
| """ | |
| Normalize landmarks relative to wrist position and hand size. | |
| Returns 63 values (21 landmarks ร 3 coords) for one hand. | |
| """ | |
| if landmarks is None: | |
| return [0.0] * 63 | |
| coords = [] | |
| for lm in landmarks: | |
| coords.extend([lm.x, lm.y, lm.z]) | |
| coords = np.array(coords, dtype=np.float32) | |
| # Normalize relative to wrist (first landmark) | |
| wrist = coords[:3].copy() | |
| for i in range(21): | |
| coords[i*3:i*3+3] -= wrist | |
| # Scale by hand size (distance from wrist to middle finger MCP) | |
| middle_mcp = coords[9*3:9*3+3] # Landmark 9 | |
| hand_size = np.linalg.norm(middle_mcp) | |
| if hand_size > 0.001: | |
| coords /= hand_size | |
| return coords.tolist() | |
| def process_hands(self, results): | |
| """ | |
| Process MediaPipe results into model input. | |
| Returns: (features, hand_info) | |
| - features: numpy array of shape (input_size,) | |
| - hand_info: dict with orientation info for display | |
| """ | |
| features = [0.0] * self.input_size | |
| hand_info = { | |
| 'hand1': None, | |
| 'hand2': None, | |
| 'hand1_orientation': None, | |
| 'hand2_orientation': None | |
| } | |
| if results.multi_hand_landmarks is None: | |
| return np.array(features, dtype=np.float32), hand_info | |
| hands_data = [] | |
| for i, (hand_landmarks, handedness) in enumerate( | |
| zip(results.multi_hand_landmarks, results.multi_handedness) | |
| ): | |
| hand_label = handedness.classification[0].label | |
| landmarks = hand_landmarks.landmark | |
| # Normalize landmarks | |
| normalized = self.normalize_landmarks(landmarks) | |
| # Calculate orientation | |
| is_palm, is_left = HandOrientationCalculator.calculate_orientation( | |
| landmarks, hand_label | |
| ) | |
| hands_data.append({ | |
| 'landmarks': normalized, | |
| 'is_palm': is_palm, | |
| 'is_left': is_left, | |
| 'label': hand_label, | |
| 'raw_landmarks': landmarks | |
| }) | |
| # Sort by x position (left to right in image) | |
| if len(hands_data) > 0: | |
| hands_data.sort( | |
| key=lambda h: h['raw_landmarks'][0].x | |
| ) | |
| # Fill features for hand 1 | |
| if len(hands_data) >= 1: | |
| h1 = hands_data[0] | |
| features[0:63] = h1['landmarks'] | |
| hand_info['hand1'] = h1['label'] | |
| hand_info['hand1_orientation'] = 'Palm' if h1['is_palm'] == 1.0 else 'Back' | |
| if self.has_orientation: | |
| features[126] = h1['is_palm'] | |
| features[127] = h1['is_left'] | |
| # Fill features for hand 2 | |
| if len(hands_data) >= 2: | |
| h2 = hands_data[1] | |
| features[63:126] = h2['landmarks'] | |
| hand_info['hand2'] = h2['label'] | |
| hand_info['hand2_orientation'] = 'Palm' if h2['is_palm'] == 1.0 else 'Back' | |
| if self.has_orientation: | |
| features[128] = h2['is_palm'] | |
| features[129] = h2['is_left'] | |
| elif self.has_orientation: | |
| features[128] = -1.0 | |
| features[129] = -1.0 | |
| return np.array(features, dtype=np.float32), hand_info | |
| # ============================================================================ | |
| # PREDICTION SMOOTHER | |
| # ============================================================================ | |
| class PredictionSmoother: | |
| """Smooth predictions over multiple frames to reduce flickering.""" | |
| def __init__(self, window_size=5, num_classes=35): | |
| self.window_size = window_size | |
| self.num_classes = num_classes | |
| self.predictions = deque(maxlen=window_size) | |
| def add_prediction(self, probs): | |
| """Add a new prediction probability distribution.""" | |
| self.predictions.append(probs) | |
| def get_smoothed_prediction(self): | |
| """Get averaged prediction over the window.""" | |
| if len(self.predictions) == 0: | |
| return None, 0.0 | |
| avg_probs = np.mean(list(self.predictions), axis=0) | |
| pred_class = np.argmax(avg_probs) | |
| confidence = avg_probs[pred_class] | |
| return pred_class, confidence | |
| def clear(self): | |
| """Clear prediction history.""" | |
| self.predictions.clear() | |
| # ============================================================================ | |
| # MAIN TESTER CLASS | |
| # ============================================================================ | |
| class ISLModelTester: | |
| """Main class for testing the ISL model with webcam.""" | |
| def __init__(self): | |
| self.model_path = MODEL_PATH | |
| self.labels = DEFAULT_LABELS | |
| self.input_size = 130 | |
| # Try to load model and labels | |
| self._load_model() | |
| self._load_labels() | |
| # Initialize components | |
| self.processor = LandmarkProcessor(self.input_size) | |
| self.smoother = PredictionSmoother(SMOOTHING_WINDOW, len(self.labels)) | |
| # Initialize MediaPipe | |
| self.mp_hands = mp.solutions.hands | |
| self.mp_draw = mp.solutions.drawing_utils | |
| self.hands = self.mp_hands.Hands( | |
| static_image_mode=False, | |
| max_num_hands=2, | |
| min_detection_confidence=0.7, | |
| min_tracking_confidence=0.5 | |
| ) | |
| # State | |
| self.confidence_threshold = CONFIDENCE_THRESHOLD | |
| self.show_landmarks = SHOW_LANDMARKS | |
| def _load_model(self): | |
| """Load TFLite model.""" | |
| # Try full model first | |
| if os.path.exists(MODEL_PATH): | |
| self.model_path = MODEL_PATH | |
| elif os.path.exists(FALLBACK_MODEL): | |
| print(f"โ ๏ธ {MODEL_PATH} not found, using {FALLBACK_MODEL}") | |
| self.model_path = FALLBACK_MODEL | |
| else: | |
| raise FileNotFoundError(f"No model found! Train the model first.") | |
| print(f"๐ฆ Loading model: {self.model_path}") | |
| self.interpreter = tf.lite.Interpreter(model_path=self.model_path) | |
| self.interpreter.allocate_tensors() | |
| self.input_details = self.interpreter.get_input_details() | |
| self.output_details = self.interpreter.get_output_details() | |
| # Get input size from model | |
| self.input_size = self.input_details[0]['shape'][1] | |
| print(f" Input size: {self.input_size}") | |
| def _load_labels(self): | |
| """Load labels from JSON.""" | |
| labels_path = LABELS_PATH if os.path.exists(LABELS_PATH) else FALLBACK_LABELS | |
| if os.path.exists(labels_path): | |
| with open(labels_path, 'r') as f: | |
| config = json.load(f) | |
| self.labels = config.get('labels', DEFAULT_LABELS) | |
| print(f" Labels loaded: {len(self.labels)} classes") | |
| else: | |
| print(f" Using default labels: {len(self.labels)} classes") | |
| def predict(self, features): | |
| """Run inference on features.""" | |
| input_data = features.reshape(1, -1).astype(np.float32) | |
| self.interpreter.set_tensor(self.input_details[0]['index'], input_data) | |
| self.interpreter.invoke() | |
| output = self.interpreter.get_tensor(self.output_details[0]['index']) | |
| return output[0] | |
| def draw_info_box(self, frame, prediction, confidence, hand_info): | |
| """Draw prediction info box on frame.""" | |
| h, w = frame.shape[:2] | |
| # Draw semi-transparent background | |
| overlay = frame.copy() | |
| cv2.rectangle(overlay, (10, 10), (300, 150), COLOR_BOX, -1) | |
| cv2.addWeighted(overlay, 0.7, frame, 0.3, 0, frame) | |
| # Draw prediction | |
| if prediction is not None and confidence >= self.confidence_threshold: | |
| label = self.labels[prediction] | |
| color = (0, 255, 0) if confidence > 0.8 else (0, 255, 255) | |
| cv2.putText(frame, f"Sign: {label}", (20, 50), | |
| cv2.FONT_HERSHEY_SIMPLEX, 1.2, color, 3) | |
| cv2.putText(frame, f"Confidence: {confidence*100:.1f}%", (20, 85), | |
| cv2.FONT_HERSHEY_SIMPLEX, 0.7, COLOR_TEXT, 2) | |
| else: | |
| cv2.putText(frame, "No sign detected", (20, 50), | |
| cv2.FONT_HERSHEY_SIMPLEX, 0.8, (100, 100, 100), 2) | |
| # Draw hand info | |
| y_offset = 115 | |
| if hand_info['hand1']: | |
| orient = hand_info['hand1_orientation'] or "?" | |
| cv2.putText(frame, f"Hand 1: {hand_info['hand1']} ({orient})", | |
| (20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1) | |
| y_offset += 20 | |
| if hand_info['hand2']: | |
| orient = hand_info['hand2_orientation'] or "?" | |
| cv2.putText(frame, f"Hand 2: {hand_info['hand2']} ({orient})", | |
| (20, y_offset), cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLOR_TEXT, 1) | |
| # Draw controls hint | |
| cv2.putText(frame, "Q: Quit | S: Screenshot | L: Toggle landmarks", | |
| (10, h - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (150, 150, 150), 1) | |
| return frame | |
| def draw_landmarks(self, frame, results, hand_info): | |
| """Draw hand landmarks with orientation-based coloring.""" | |
| if results.multi_hand_landmarks is None: | |
| return frame | |
| for i, hand_landmarks in enumerate(results.multi_hand_landmarks): | |
| # Choose color based on orientation | |
| if i == 0 and hand_info['hand1_orientation']: | |
| color = COLOR_PALM if hand_info['hand1_orientation'] == 'Palm' else COLOR_BACK | |
| elif i == 1 and hand_info['hand2_orientation']: | |
| color = COLOR_PALM if hand_info['hand2_orientation'] == 'Palm' else COLOR_BACK | |
| else: | |
| color = (200, 200, 200) | |
| # Draw connections | |
| self.mp_draw.draw_landmarks( | |
| frame, hand_landmarks, self.mp_hands.HAND_CONNECTIONS, | |
| self.mp_draw.DrawingSpec(color=color, thickness=2, circle_radius=2), | |
| self.mp_draw.DrawingSpec(color=color, thickness=2) | |
| ) | |
| return frame | |
| def run(self): | |
| """Main loop for webcam testing.""" | |
| print("\n" + "=" * 60) | |
| print("ISL MODEL TEST WITH MEDIAPIPE") | |
| print("=" * 60) | |
| print("\n๐ท Opening webcam...") | |
| cap = cv2.VideoCapture(0) | |
| if not cap.isOpened(): | |
| print("โ Error: Could not open webcam!") | |
| return | |
| # Set camera properties | |
| cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) | |
| cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) | |
| cap.set(cv2.CAP_PROP_FPS, 30) | |
| print("โ Webcam opened successfully") | |
| print("\n๐ฎ Controls:") | |
| print(" Press 'q' to quit") | |
| print(" Press 's' to save screenshot") | |
| print(" Press 'l' to toggle landmarks") | |
| print(" Press 'c' to cycle confidence threshold") | |
| print(" Press 'r' to reset smoother") | |
| print("\n" + "=" * 60 + "\n") | |
| screenshot_count = 0 | |
| while True: | |
| ret, frame = cap.read() | |
| if not ret: | |
| print("โ Error reading frame") | |
| break | |
| # Flip for mirror effect | |
| frame = cv2.flip(frame, 1) | |
| # Convert to RGB for MediaPipe | |
| rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) | |
| # Process with MediaPipe | |
| results = self.hands.process(rgb_frame) | |
| # Extract features | |
| features, hand_info = self.processor.process_hands(results) | |
| # Run inference if hand detected | |
| prediction = None | |
| confidence = 0.0 | |
| if hand_info['hand1'] is not None: | |
| probs = self.predict(features) | |
| self.smoother.add_prediction(probs) | |
| prediction, confidence = self.smoother.get_smoothed_prediction() | |
| else: | |
| self.smoother.clear() | |
| # Draw landmarks if enabled | |
| if self.show_landmarks: | |
| frame = self.draw_landmarks(frame, results, hand_info) | |
| # Draw info box | |
| frame = self.draw_info_box(frame, prediction, confidence, hand_info) | |
| # Show frame | |
| cv2.imshow('ISL Model Test', frame) | |
| # Handle key presses | |
| key = cv2.waitKey(1) & 0xFF | |
| if key == ord('q'): | |
| print("\n๐ Exiting...") | |
| break | |
| elif key == ord('s'): | |
| filename = f"screenshot_{screenshot_count}.png" | |
| cv2.imwrite(filename, frame) | |
| print(f"๐ธ Screenshot saved: {filename}") | |
| screenshot_count += 1 | |
| elif key == ord('l'): | |
| self.show_landmarks = not self.show_landmarks | |
| print(f"๐ฏ Landmarks: {'ON' if self.show_landmarks else 'OFF'}") | |
| elif key == ord('c'): | |
| # Cycle threshold: 0.5 -> 0.7 -> 0.9 -> 0.3 -> 0.5 | |
| thresholds = [0.3, 0.5, 0.7, 0.9] | |
| idx = thresholds.index(self.confidence_threshold) if self.confidence_threshold in thresholds else 0 | |
| self.confidence_threshold = thresholds[(idx + 1) % len(thresholds)] | |
| print(f"๐ Confidence threshold: {self.confidence_threshold}") | |
| elif key == ord('r'): | |
| self.smoother.clear() | |
| print("๐ Smoother reset") | |
| cap.release() | |
| cv2.destroyAllWindows() | |
| print("\nโ Test complete!") | |
| # ============================================================================ | |
| # BATCH TEST MODE | |
| # ============================================================================ | |
| def test_on_images(image_folder): | |
| """Test model on a folder of images.""" | |
| print("\n" + "=" * 60) | |
| print("BATCH IMAGE TEST") | |
| print("=" * 60) | |
| tester = ISLModelTester() | |
| results = [] | |
| for filename in os.listdir(image_folder): | |
| if filename.lower().endswith(('.png', '.jpg', '.jpeg')): | |
| filepath = os.path.join(image_folder, filename) | |
| # Load image | |
| frame = cv2.imread(filepath) | |
| if frame is None: | |
| continue | |
| # Process | |
| rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) | |
| mp_results = tester.hands.process(rgb_frame) | |
| features, hand_info = tester.processor.process_hands(mp_results) | |
| if hand_info['hand1'] is not None: | |
| probs = tester.predict(features) | |
| pred_class = np.argmax(probs) | |
| confidence = probs[pred_class] | |
| results.append({ | |
| 'file': filename, | |
| 'prediction': tester.labels[pred_class], | |
| 'confidence': confidence | |
| }) | |
| print(f" {filename}: {tester.labels[pred_class]} ({confidence*100:.1f}%)") | |
| else: | |
| print(f" {filename}: No hand detected") | |
| return results | |
| # ============================================================================ | |
| # MAIN | |
| # ============================================================================ | |
| def main(): | |
| import sys | |
| if len(sys.argv) > 1: | |
| # Batch mode on folder | |
| folder = sys.argv[1] | |
| if os.path.isdir(folder): | |
| test_on_images(folder) | |
| else: | |
| print(f"โ Folder not found: {folder}") | |
| else: | |
| # Webcam mode | |
| tester = ISLModelTester() | |
| tester.run() | |
| if __name__ == "__main__": | |
| main() | |