Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import cv2 | |
| import numpy as np | |
| import tensorflow as tf | |
| from streamlit_webrtc import webrtc_streamer, VideoTransformerBase, RTCConfiguration | |
| import av | |
| import mediapipe as mp | |
| import google.generativeai as genai | |
| from collections import deque | |
| import time | |
| import json | |
| # Configure page | |
| st.set_page_config( | |
| page_title="🤟 AI Sign Language Interpreter", | |
| page_icon="🤟", | |
| layout="wide", | |
| initial_sidebar_state="expanded" | |
| ) | |
| # Load custom CSS | |
| st.markdown(""" | |
| <style> | |
| .main-header { | |
| background: linear-gradient(90deg, #667eea 0%, #764ba2 100%); | |
| padding: 2rem; | |
| border-radius: 10px; | |
| color: white; | |
| text-align: center; | |
| margin-bottom: 2rem; | |
| } | |
| .feature-card { | |
| background: #ffffff; | |
| border: 1px solid #e0e0e0; | |
| padding: 1.5rem; | |
| border-radius: 10px; | |
| border-left: 4px solid #667eea; | |
| margin: 1rem 0; | |
| color: #1a1a1a; | |
| font-weight: 600; | |
| box-shadow: 0 2px 4px rgba(0,0,0,0.1); | |
| } | |
| .prediction-box { | |
| background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); | |
| color: white; | |
| padding: 1.5rem; | |
| border-radius: 10px; | |
| text-align: center; | |
| font-size: 1.2rem; | |
| margin: 1rem 0; | |
| } | |
| .confidence-meter { | |
| background: #e9ecef; | |
| border-radius: 10px; | |
| padding: 0.5rem; | |
| margin: 0.5rem 0; | |
| } | |
| .sentence-builder { | |
| background: #ffffff; | |
| border: 2px solid #e0e0e0; | |
| border-radius: 10px; | |
| padding: 1rem; | |
| margin: 1rem 0; | |
| min-height: 100px; | |
| color: #1a1a1a; | |
| font-weight: 600; | |
| font-size: 1.1rem; | |
| } | |
| .control-panel { | |
| background: #ffffff; | |
| border: 2px solid #dee2e6; | |
| border-radius: 10px; | |
| padding: 1rem; | |
| margin: 1rem 0; | |
| color: #1a1a1a; | |
| font-weight: 600; | |
| box-shadow: 0 2px 4px rgba(0,0,0,0.1); | |
| } | |
| </style> | |
| """, unsafe_allow_html=True) | |
| # Initialize Gemini AI | |
| def initialize_gemini(): | |
| try: | |
| genai.configure(api_key=st.secrets["GEMINI_API_KEY"]) | |
| model = genai.GenerativeModel('gemini-pro') | |
| return model | |
| except Exception as e: | |
| st.error(f"Failed to initialize Gemini AI: {e}") | |
| return None | |
| # Load model and labels | |
| def load_model_and_labels(): | |
| try: | |
| interpreter = tf.lite.Interpreter(model_path="models/model.tflite") | |
| interpreter.allocate_tensors() | |
| with open("models/labels.txt", "r") as f: | |
| labels = [line.strip() for line in f.readlines()] | |
| return interpreter, labels | |
| except Exception as e: | |
| st.error(f"Error loading model: {e}") | |
| return None, [] | |
| # Initialize MediaPipe | |
| def initialize_mediapipe(): | |
| mp_hands = mp.solutions.hands | |
| hands = mp_hands.Hands( | |
| static_image_mode=False, | |
| max_num_hands=2, | |
| min_detection_confidence=0.7, | |
| min_tracking_confidence=0.5 | |
| ) | |
| mp_drawing = mp.solutions.drawing_utils | |
| return hands, mp_drawing, mp_hands | |
| class EnhancedSignInterpreter(VideoTransformerBase): | |
| def __init__(self, confidence_threshold=0.7, sequence_length=10): | |
| self.interpreter, self.labels = load_model_and_labels() | |
| self.hands, self.mp_drawing, self.mp_hands = initialize_mediapipe() | |
| self.gemini_model = initialize_gemini() | |
| # Enhanced features | |
| self.confidence_threshold = confidence_threshold | |
| self.sequence_length = sequence_length | |
| self.gesture_sequence = deque(maxlen=sequence_length) | |
| self.last_prediction = "" | |
| self.last_confidence = 0.0 | |
| self.sentence_builder = [] | |
| self.last_gesture_time = time.time() | |
| self.gesture_hold_time = 1.5 # seconds to hold gesture for confirmation | |
| # Prediction smoothing | |
| self.prediction_buffer = deque(maxlen=5) | |
| def extract_landmarks(self, image): | |
| """Extract hand landmarks using MediaPipe""" | |
| image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| results = self.hands.process(image_rgb) | |
| if results.multi_hand_landmarks: | |
| landmarks = [] | |
| for hand_landmarks in results.multi_hand_landmarks: | |
| for landmark in hand_landmarks.landmark: | |
| landmarks.extend([landmark.x, landmark.y, landmark.z]) | |
| # Pad or truncate to 63 features (21 landmarks * 3 coordinates) | |
| if len(landmarks) < 63: | |
| landmarks.extend([0.0] * (63 - len(landmarks))) | |
| else: | |
| landmarks = landmarks[:63] | |
| return np.array(landmarks, dtype=np.float32) | |
| return None | |
| def predict_gesture(self, landmarks): | |
| """Predict gesture from landmarks""" | |
| if self.interpreter is None or landmarks is None: | |
| return "No gesture", 0.0 | |
| try: | |
| input_details = self.interpreter.get_input_details() | |
| output_details = self.interpreter.get_output_details() | |
| # Reshape landmarks for model input | |
| input_data = landmarks.reshape(1, -1) | |
| self.interpreter.set_tensor(input_details[0]['index'], input_data) | |
| self.interpreter.invoke() | |
| output_data = self.interpreter.get_tensor(output_details[0]['index']) | |
| predicted_class = np.argmax(output_data) | |
| confidence = float(np.max(output_data)) | |
| if predicted_class < len(self.labels): | |
| return self.labels[predicted_class], confidence | |
| else: | |
| return "Unknown", confidence | |
| except Exception as e: | |
| return f"Error: {str(e)}", 0.0 | |
| def smooth_prediction(self, prediction, confidence): | |
| """Smooth predictions using buffer""" | |
| self.prediction_buffer.append((prediction, confidence)) | |
| # Get most common prediction in buffer | |
| predictions = [p[0] for p in self.prediction_buffer] | |
| confidences = [p[1] for p in self.prediction_buffer] | |
| # Find most frequent prediction | |
| unique_predictions = list(set(predictions)) | |
| if unique_predictions: | |
| most_common = max(unique_predictions, key=predictions.count) | |
| avg_confidence = np.mean([c for p, c in self.prediction_buffer if p == most_common]) | |
| return most_common, avg_confidence | |
| return prediction, confidence | |
| def update_sentence_builder(self, gesture, confidence): | |
| """Update sentence builder with confirmed gestures""" | |
| current_time = time.time() | |
| cooldown_period = 0.5 # Time to wait before accepting new gestures | |
| # Check if the gesture is valid and meets confidence threshold | |
| is_valid_gesture = (gesture not in ["No gesture", "Unknown"] and | |
| confidence > self.confidence_threshold) | |
| # Time elapsed since last gesture | |
| time_elapsed = current_time - self.last_gesture_time | |
| if is_valid_gesture: | |
| # Case 1: New gesture detected | |
| if gesture != self.last_prediction: | |
| self.last_prediction = gesture | |
| self.last_confidence = confidence | |
| self.last_gesture_time = current_time | |
| # Case 2: Gesture held long enough and cooldown period passed | |
| elif (time_elapsed > self.gesture_hold_time and | |
| (not self.sentence_builder or | |
| time_elapsed > self.gesture_hold_time + cooldown_period)): | |
| if not self.sentence_builder or self.sentence_builder[-1] != gesture: | |
| self.sentence_builder.append(gesture) | |
| self.last_gesture_time = current_time | |
| self.last_prediction = "" # Reset prediction to allow new gestures | |
| else: | |
| # Reset prediction if gesture is invalid or confidence is low | |
| if time_elapsed > cooldown_period: | |
| self.last_prediction = "" | |
| def generate_ai_interpretation(self, gestures): | |
| """Generate AI interpretation using Gemini""" | |
| if not gestures or not self.gemini_model: | |
| return "No interpretation available" | |
| try: | |
| gesture_text = " ".join(gestures) | |
| prompt = f""" | |
| Convert these sign language gestures into a natural, meaningful sentence: {gesture_text} | |
| Guidelines: | |
| - Form complete, grammatically correct sentences | |
| - Add appropriate articles, prepositions, and conjunctions | |
| - Consider context and common sign language patterns | |
| - If it's a sequence of letters, try to form words | |
| - If it's numbers, consider if they represent quantities, dates, or codes | |
| - Make the output conversational and natural | |
| Gestures: {gesture_text} | |
| Natural interpretation: | |
| """ | |
| response = self.gemini_model.generate_content(prompt) | |
| return response.text.strip() | |
| except Exception as e: | |
| return f"AI interpretation error: {str(e)}" | |
| def recv(self, frame): | |
| image = frame.to_ndarray(format="bgr24") | |
| # Extract landmarks | |
| landmarks = self.extract_landmarks(image) | |
| if landmarks is not None: | |
| # Predict gesture | |
| gesture, confidence = self.predict_gesture(landmarks) | |
| # Smooth prediction | |
| gesture, confidence = self.smooth_prediction(gesture, confidence) | |
| # Update sentence builder | |
| self.update_sentence_builder(gesture, confidence) | |
| # Draw hand landmarks | |
| image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) | |
| results = self.hands.process(image_rgb) | |
| if results.multi_hand_landmarks: | |
| for hand_landmarks in results.multi_hand_landmarks: | |
| self.mp_drawing.draw_landmarks( | |
| image, hand_landmarks, self.mp_hands.HAND_CONNECTIONS) | |
| # Draw prediction info | |
| color = (0, 255, 0) if confidence > self.confidence_threshold else (0, 165, 255) | |
| cv2.putText(image, f"{gesture} ({confidence:.2f})", | |
| (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, color, 2) | |
| # Store current prediction for UI | |
| st.session_state.current_gesture = gesture | |
| st.session_state.current_confidence = confidence | |
| st.session_state.sentence_builder = self.sentence_builder.copy() | |
| else: | |
| st.session_state.current_gesture = "No hand detected" | |
| st.session_state.current_confidence = 0.0 | |
| return av.VideoFrame.from_ndarray(image, format="bgr24") | |
| def main(): | |
| # Header | |
| st.markdown(""" | |
| <div class="main-header"> | |
| <h1>🤟 Enhanced AI Sign Language Interpreter</h1> | |
| <p>Advanced real-time interpretation with AI-powered sentence formation</p> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Initialize session state | |
| if 'current_gesture' not in st.session_state: | |
| st.session_state.current_gesture = "Ready to start" | |
| st.session_state.current_confidence = 0.0 | |
| st.session_state.sentence_builder = [] | |
| st.session_state.ai_interpretation = "" | |
| # Sidebar controls | |
| with st.sidebar: | |
| st.markdown("### 🎛️ Control Panel") | |
| confidence_threshold = st.slider( | |
| "Confidence Threshold", | |
| min_value=0.1, | |
| max_value=1.0, | |
| value=0.7, | |
| step=0.1, | |
| help="Minimum confidence required for gesture recognition" | |
| ) | |
| sequence_length = st.slider( | |
| "Sequence Memory", | |
| min_value=5, | |
| max_value=20, | |
| value=10, | |
| help="Number of gestures to remember for sentence building" | |
| ) | |
| gesture_hold_time = st.slider( | |
| "Gesture Hold Time (seconds)", | |
| min_value=0.5, | |
| max_value=3.0, | |
| value=1.5, | |
| step=0.1, | |
| help="Time to hold gesture for confirmation" | |
| ) | |
| st.markdown("### 🎯 Quick Actions") | |
| if st.button("🗑️ Clear Sentence", use_container_width=True): | |
| st.session_state.sentence_builder = [] | |
| st.session_state.ai_interpretation = "" | |
| if st.button("🤖 Generate AI Interpretation", use_container_width=True): | |
| if st.session_state.sentence_builder: | |
| interpreter = EnhancedSignInterpreter() | |
| st.session_state.ai_interpretation = interpreter.generate_ai_interpretation( | |
| st.session_state.sentence_builder | |
| ) | |
| # Display supported gestures | |
| st.markdown("### 📋 Supported Gestures") | |
| st.markdown("**Numbers:** 1, 2, 4, 5, 6, 7, 8") | |
| st.markdown("**Letters:** A-X (except E)") | |
| # Main content | |
| col1, col2 = st.columns([2, 1]) | |
| with col1: | |
| st.markdown("### 📹 Live Camera Feed") | |
| # WebRTC configuration | |
| rtc_configuration = RTCConfiguration({ | |
| "iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}] | |
| }) | |
| # Video streamer | |
| webrtc_ctx = webrtc_streamer( | |
| key="enhanced-sign-interpreter", | |
| video_transformer_factory=lambda: EnhancedSignInterpreter( | |
| confidence_threshold=confidence_threshold, | |
| sequence_length=sequence_length | |
| ), | |
| rtc_configuration=rtc_configuration, | |
| media_stream_constraints={"video": True, "audio": False}, | |
| async_processing=True, | |
| ) | |
| with col2: | |
| st.markdown("### 🎯 Recognition Results") | |
| # Current prediction | |
| st.markdown(f""" | |
| <div class="prediction-box"> | |
| <h3>{st.session_state.current_gesture}</h3> | |
| <div class="confidence-meter"> | |
| <div style="background: {'#28a745' if st.session_state.current_confidence > confidence_threshold else '#ffc107'}; | |
| width: {st.session_state.current_confidence * 100}%; | |
| height: 20px; | |
| border-radius: 10px; | |
| transition: all 0.3s ease;"> | |
| </div> | |
| <small>Confidence: {st.session_state.current_confidence:.1%}</small> | |
| </div> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Sentence builder | |
| st.markdown("### 📝 Sentence Builder") | |
| sentence_text = " → ".join(st.session_state.sentence_builder) if st.session_state.sentence_builder else "Start making gestures..." | |
| st.markdown(f""" | |
| <div class="sentence-builder"> | |
| <strong>Gestures:</strong><br> | |
| {sentence_text} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # AI Interpretation | |
| if st.session_state.ai_interpretation: | |
| st.markdown("### 🤖 AI Interpretation") | |
| st.markdown(f""" | |
| <div class="feature-card"> | |
| <strong>Natural Language:</strong><br> | |
| {st.session_state.ai_interpretation} | |
| </div> | |
| """, unsafe_allow_html=True) | |
| # Statistics and tips | |
| st.markdown("---") | |
| col1, col2, col3 = st.columns(3) | |
| with col1: | |
| st.markdown(""" | |
| <div class="feature-card"> | |
| <h4>💡 Tips for Better Recognition</h4> | |
| <ul> | |
| <li>Ensure good lighting</li> | |
| <li>Keep hand centered in frame</li> | |
| <li>Hold gestures steady</li> | |
| <li>Avoid cluttered backgrounds</li> | |
| </ul> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with col2: | |
| st.markdown(f""" | |
| <div class="feature-card"> | |
| <h4>📊 Session Statistics</h4> | |
| <ul> | |
| <li>Gestures Captured: {len(st.session_state.sentence_builder)}</li> | |
| <li>Current Confidence: {st.session_state.current_confidence:.1%}</li> | |
| <li>Threshold: {confidence_threshold:.1%}</li> | |
| </ul> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| with col3: | |
| st.markdown(""" | |
| <div class="feature-card"> | |
| <h4>🚀 Enhanced Features</h4> | |
| <ul> | |
| <li>Confidence threshold control</li> | |
| <li>Gesture sequence memory</li> | |
| <li>AI sentence formation</li> | |
| <li>Real-time smoothing</li> | |
| </ul> | |
| </div> | |
| """, unsafe_allow_html=True) | |
| if __name__ == "__main__": | |
| main() | |