Spaces:

KrishDev15
/

Sign_intrepreter

Sleeping

Upload 30 files

1f4e4ae verified over 1 year ago

13 kB

import streamlit as st
from streamlit_webrtc import webrtc_streamer, VideoTransformerBase, RTCConfiguration
import cv2
import numpy as np
import mediapipe as mp
import tensorflow as tf
import threading
import time
import google.generativeai as genai
from av import VideoFrame

# Page config
st.set_page_config(
    page_title="AI Sign Language Interpreter",
    page_icon="🤟",
    layout="wide",
    initial_sidebar_state="expanded"
)

# Custom CSS
st.markdown("""
<style>
    .main {
        background-color: #f8f9fa;
    }
    .stApp {
        max-width: 1200px;
        margin: 0 auto;
    }
    .prediction-box {
        background-color: white;
        padding: 2rem;
        border-radius: 10px;
        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
        margin-bottom: 1rem;
    }
    .interpretation-box {
        background-color: #f0f7ff;
        padding: 2rem;
        border-radius: 10px;
        border: 1px solid #cce5ff;
        margin-bottom: 1rem;
    }
    .stButton>button {
        background-color: #4c51bf;
        color: white;
        font-weight: bold;
    }
    .stButton>button:hover {
        background-color: #434190;
    }
</style>
""", unsafe_allow_html=True)

# Initialize Gemini AI
@st.cache_resource
def initialize_gemini():
    try:
        genai.configure(api_key=st.secrets["GEMINI_API_KEY"])
        model = genai.GenerativeModel('gemini-pro')
        return model
    except Exception as e:
        st.error(f"Error initializing Gemini AI: {e}")
        return None

# RTC Configuration
RTC_CONFIGURATION = RTCConfiguration({
    "iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]
})

# Global variables for thread-safe prediction sharing
prediction_lock = threading.Lock()
current_prediction = {
    "gesture": "Show your hand",
    "confidence": 0.0,
    "interpretation": ""
}

# Load model and labels
@st.cache_resource
def load_model():
    try:
        interpreter = tf.lite.Interpreter(model_path="models/model.tflite")
        interpreter.allocate_tensors()
        
        with open("models/labels.txt", 'r') as f:
            labels = [line.strip() for line in f.readlines()]
        
        return interpreter, labels
    except Exception as e:
        st.error(f"Error loading model: {e}")
        return None, []

interpreter, labels = load_model()
gemini_model = initialize_gemini()

# MediaPipe setup
@st.cache_resource
def get_mediapipe_hands():
    mp_hands = mp.solutions.hands
    mp_drawing = mp.solutions.drawing_utils
    hands = mp_hands.Hands(
        static_image_mode=False,
        max_num_hands=1,
        min_detection_confidence=0.7,
        min_tracking_confidence=0.5
    )
    return mp_hands, mp_drawing, hands

mp_hands, mp_drawing, hands = get_mediapipe_hands()

def extract_hand_features(landmarks):
    """Extract features from hand landmarks."""
    try:
        landmarks_array = np.array([(lm.x, lm.y, lm.z) for lm in landmarks.landmark])
        if len(landmarks_array) != 21:
            return np.zeros(63)
        
        wrist = landmarks_array[0]
        normalized = landmarks_array - wrist
        hand_size = np.linalg.norm(landmarks_array[12] - wrist)
        
        if hand_size > 0:
            normalized = normalized / hand_size
        
        features = normalized.flatten()
        if len(features) != 63:
            features = np.pad(features, (0, max(0, 63 - len(features))))[:63]
        
        return features
    except Exception as e:
        print(f"Error extracting features: {e}")
        return np.zeros(63)

def predict_gesture(features):
    """Predict gesture from features."""
    if interpreter is None:
        return "Model not loaded", 0.0
    
    try:
        input_data = np.expand_dims(features, axis=0).astype(np.float32)
        input_details = interpreter.get_input_details()
        output_details = interpreter.get_output_details()
        
        interpreter.set_tensor(input_details[0]['index'], input_data)
        interpreter.invoke()
        
        output = interpreter.get_tensor(output_details[0]['index'])
        predicted_class = np.argmax(output[0])
        
        exp_output = np.exp(output[0] - np.max(output[0]))
        softmax_output = exp_output / np.sum(exp_output)
        confidence = float(softmax_output[predicted_class])
        
        gesture = labels[predicted_class] if predicted_class < len(labels) else str(predicted_class)
        return gesture, confidence
    except Exception as e:
        print(f"Prediction error: {e}")
        return "Error", 0.0

def get_gesture_interpretation(gesture_sequence):
    """Get interpretation from Gemini AI."""
    if not gemini_model or not gesture_sequence:
        return ""
    
    try:
        prompt = f"""
        As a sign language interpreter, analyze this sequence of gestures and provide a natural interpretation:
        Gestures detected: {', '.join(gesture_sequence)}
        
        Provide a clear and concise interpretation. If the gestures form a word or phrase, explain its meaning.
        If they're individual letters/numbers, format them clearly.
        """
        
        response = gemini_model.generate_content(prompt)
        return response.text
    except Exception as e:
        print(f"Gemini AI error: {e}")
        return ""

class GestureRecognizer(VideoTransformerBase):
    def __init__(self):
        self.frame_count = 0
        self.last_prediction_time = 0
        self.prediction_interval = 0.2
        self.gesture_history = []
        self.last_interpretation_time = 0
        self.interpretation_interval = 2.0

    def recv(self, frame: VideoFrame) -> VideoFrame:
        image = frame.to_ndarray(format="bgr24")
        self.frame_count += 1
        current_time = time.time()
        
        if current_time - self.last_prediction_time > self.prediction_interval:
            self.last_prediction_time = current_time
            
            try:
                rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
                results = hands.process(rgb_image)
                
                if results.multi_hand_landmarks:
                    hand_landmarks = results.multi_hand_landmarks[0]
                    
                    mp_drawing.draw_landmarks(
                        image, hand_landmarks, mp_hands.HAND_CONNECTIONS,
                        mp_drawing.DrawingSpec(color=(0, 0, 255), thickness=2, circle_radius=2),
                        mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2)
                    )
                    
                    features = extract_hand_features(hand_landmarks)
                    gesture, confidence = predict_gesture(features)
                    
                    if confidence > 0.5:
                        self.gesture_history.append(gesture)
                        self.gesture_history = self.gesture_history[-10:]  # Keep last 10 gestures
                    
                    with prediction_lock:
                        current_prediction["gesture"] = gesture
                        current_prediction["confidence"] = confidence
                        
                        # Update interpretation periodically
                        if current_time - self.last_interpretation_time > self.interpretation_interval:
                            self.last_interpretation_time = current_time
                            interpretation = get_gesture_interpretation(self.gesture_history)
                            if interpretation:
                                current_prediction["interpretation"] = interpretation
                else:
                    with prediction_lock:
                        current_prediction["gesture"] = "No hand detected"
                        current_prediction["confidence"] = 0.0
            except Exception as e:
                print(f"Processing error: {e}")
                with prediction_lock:
                    current_prediction["gesture"] = "Processing error"
                    current_prediction["confidence"] = 0.0
        
        # Add text overlay
        with prediction_lock:
            display_text = current_prediction["gesture"]
            confidence = current_prediction["confidence"]
        
        color = (0, 255, 0) if confidence > 0.5 else (0, 0, 255)
        cv2.putText(
            image,
            f"{display_text} ({confidence:.2f})",
            (20, 40),
            cv2.FONT_HERSHEY_SIMPLEX,
            1.0,
            color,
            2
        )
        
        return VideoFrame.from_ndarray(image, format="bgr24")

def main():
    st.title("🤟 AI Sign Language Interpreter")
    st.markdown("Transform sign language into text with real-time AI interpretation")
    
    # Create columns for layout
    col1, col2 = st.columns([2, 1])
    
    with col1:
        st.markdown("### 📹 Live Camera Feed")
        ctx = webrtc_streamer(
            key="gesture_recognition",
            video_processor_factory=GestureRecognizer,
            rtc_configuration=RTC_CONFIGURATION,
            media_stream_constraints={"video": True, "audio": False},
            async_processing=True,
        )
    
    with col2:
        st.markdown("### 🎯 Recognition Results")
        
        # Create placeholders for dynamic updates
        prediction_placeholder = st.empty()
        confidence_bar = st.empty()
        interpretation_placeholder = st.empty()
        
        if ctx.state.playing:
            while ctx.state.playing:
                with prediction_lock:
                    gesture = current_prediction["gesture"]
                    confidence = current_prediction["confidence"]
                    interpretation = current_prediction["interpretation"]
                
                # Display current gesture
                with prediction_placeholder.container():
                    st.markdown("""
                    <div class="prediction-box">
                        <h3 style="font-size: 1.5rem; margin-bottom: 0.5rem;">Current Gesture</h3>
                        <p style="font-size: 2rem; font-weight: bold; color: #4c51bf;">{}</p>
                    </div>
                    """.format(gesture), unsafe_allow_html=True)
                
                # Display confidence bar
                if confidence > 0.5 and gesture not in ["No hand detected", "Processing error", "Error"]:
                    confidence_bar.progress(confidence)
                    
                    # Display interpretation
                    if interpretation:
                        with interpretation_placeholder.container():
                            st.markdown("""
                            <div class="interpretation-box">
                                <h3 style="font-size: 1.5rem; margin-bottom: 0.5rem;">AI Interpretation</h3>
                                <p style="font-size: 1.2rem;">{}</p>
                            </div>
                            """.format(interpretation), unsafe_allow_html=True)
                else:
                    confidence_bar.progress(0)
                    if gesture == "No hand detected":
                        interpretation_placeholder.info("Show your hand to begin")
                    else:
                        interpretation_placeholder.warning("Gesture not recognized clearly")
                
                time.sleep(0.1)
        else:
            st.warning("📷 Turn on your webcam to start!")
    
    # Instructions and supported gestures
    st.markdown("---")
    
    # Display supported gestures in an organized grid
    st.markdown("### 🎯 Supported Gestures")
    
    # Numbers
    st.markdown("#### 🔢 Numbers")
    number_labels = [label for label in labels if label.isdigit()]
    if number_labels:
        cols = st.columns(len(number_labels))
        for i, label in enumerate(number_labels):
            with cols[i]:
                st.markdown(f"""
                <div style="text-align: center; padding: 1rem; background-color: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1);">
                    <h3 style="font-size: 1.5rem; color: #4c51bf;">{label}</h3>
                </div>
                """, unsafe_allow_html=True)
    
    # Letters
    st.markdown("#### 🔤 Letters")
    letter_labels = [label for label in labels if label.isalpha()]
    if letter_labels:
        # Display letters in rows of 8
        for i in range(0, len(letter_labels), 8):
            row_labels = letter_labels[i:i+8]
            cols = st.columns(len(row_labels))
            for j, label in enumerate(row_labels):
                with cols[j]:
                    st.markdown(f"""
                    <div style="text-align: center; padding: 1rem; background-color: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1);">
                        <h3 style="font-size: 1.5rem; color: #4c51bf;">{label}</h3>
                    </div>
                    """, unsafe_allow_html=True)

if __name__ == "__main__":
    main()

	import streamlit as st
	from streamlit_webrtc import webrtc_streamer, VideoTransformerBase, RTCConfiguration
	import cv2
	import numpy as np
	import mediapipe as mp
	import tensorflow as tf
	import threading
	import time
	import google.generativeai as genai
	from av import VideoFrame

	# Page config
	st.set_page_config(
	page_title="AI Sign Language Interpreter",
	page_icon="🤟",
	layout="wide",
	initial_sidebar_state="expanded"
	)

	# Custom CSS
	st.markdown("""
	<style>
	.main {
	background-color: #f8f9fa;
	}
	.stApp {
	max-width: 1200px;
	margin: 0 auto;
	}
	.prediction-box {
	background-color: white;
	padding: 2rem;
	border-radius: 10px;
	box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
	margin-bottom: 1rem;
	}
	.interpretation-box {
	background-color: #f0f7ff;
	padding: 2rem;
	border-radius: 10px;
	border: 1px solid #cce5ff;
	margin-bottom: 1rem;
	}
	.stButton>button {
	background-color: #4c51bf;
	color: white;
	font-weight: bold;
	}
	.stButton>button:hover {
	background-color: #434190;
	}
	</style>
	""", unsafe_allow_html=True)

	# Initialize Gemini AI
	@st.cache_resource
	def initialize_gemini():
	try:
	genai.configure(api_key=st.secrets["GEMINI_API_KEY"])
	model = genai.GenerativeModel('gemini-pro')
	return model
	except Exception as e:
	st.error(f"Error initializing Gemini AI: {e}")
	return None

	# RTC Configuration
	RTC_CONFIGURATION = RTCConfiguration({
	"iceServers": [{"urls": ["stun:stun.l.google.com:19302"]}]
	})

	# Global variables for thread-safe prediction sharing
	prediction_lock = threading.Lock()
	current_prediction = {
	"gesture": "Show your hand",
	"confidence": 0.0,
	"interpretation": ""
	}

	# Load model and labels
	@st.cache_resource
	def load_model():
	try:
	interpreter = tf.lite.Interpreter(model_path="models/model.tflite")
	interpreter.allocate_tensors()

	with open("models/labels.txt", 'r') as f:
	labels = [line.strip() for line in f.readlines()]

	return interpreter, labels
	except Exception as e:
	st.error(f"Error loading model: {e}")
	return None, []

	interpreter, labels = load_model()
	gemini_model = initialize_gemini()

	# MediaPipe setup
	@st.cache_resource
	def get_mediapipe_hands():
	mp_hands = mp.solutions.hands
	mp_drawing = mp.solutions.drawing_utils
	hands = mp_hands.Hands(
	static_image_mode=False,
	max_num_hands=1,
	min_detection_confidence=0.7,
	min_tracking_confidence=0.5
	)
	return mp_hands, mp_drawing, hands

	mp_hands, mp_drawing, hands = get_mediapipe_hands()

	def extract_hand_features(landmarks):
	"""Extract features from hand landmarks."""
	try:
	landmarks_array = np.array([(lm.x, lm.y, lm.z) for lm in landmarks.landmark])
	if len(landmarks_array) != 21:
	return np.zeros(63)

	wrist = landmarks_array[0]
	normalized = landmarks_array - wrist
	hand_size = np.linalg.norm(landmarks_array[12] - wrist)

	if hand_size > 0:
	normalized = normalized / hand_size

	features = normalized.flatten()
	if len(features) != 63:
	features = np.pad(features, (0, max(0, 63 - len(features))))[:63]

	return features
	except Exception as e:
	print(f"Error extracting features: {e}")
	return np.zeros(63)

	def predict_gesture(features):
	"""Predict gesture from features."""
	if interpreter is None:
	return "Model not loaded", 0.0

	try:
	input_data = np.expand_dims(features, axis=0).astype(np.float32)
	input_details = interpreter.get_input_details()
	output_details = interpreter.get_output_details()

	interpreter.set_tensor(input_details[0]['index'], input_data)
	interpreter.invoke()

	output = interpreter.get_tensor(output_details[0]['index'])
	predicted_class = np.argmax(output[0])

	exp_output = np.exp(output[0] - np.max(output[0]))
	softmax_output = exp_output / np.sum(exp_output)
	confidence = float(softmax_output[predicted_class])

	gesture = labels[predicted_class] if predicted_class < len(labels) else str(predicted_class)
	return gesture, confidence
	except Exception as e:
	print(f"Prediction error: {e}")
	return "Error", 0.0

	def get_gesture_interpretation(gesture_sequence):
	"""Get interpretation from Gemini AI."""
	if not gemini_model or not gesture_sequence:
	return ""

	try:
	prompt = f"""
	As a sign language interpreter, analyze this sequence of gestures and provide a natural interpretation:
	Gestures detected: {', '.join(gesture_sequence)}

	Provide a clear and concise interpretation. If the gestures form a word or phrase, explain its meaning.
	If they're individual letters/numbers, format them clearly.
	"""

	response = gemini_model.generate_content(prompt)
	return response.text
	except Exception as e:
	print(f"Gemini AI error: {e}")
	return ""

	class GestureRecognizer(VideoTransformerBase):
	def __init__(self):
	self.frame_count = 0
	self.last_prediction_time = 0
	self.prediction_interval = 0.2
	self.gesture_history = []
	self.last_interpretation_time = 0
	self.interpretation_interval = 2.0

	def recv(self, frame: VideoFrame) -> VideoFrame:
	image = frame.to_ndarray(format="bgr24")
	self.frame_count += 1
	current_time = time.time()

	if current_time - self.last_prediction_time > self.prediction_interval:
	self.last_prediction_time = current_time

	try:
	rgb_image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
	results = hands.process(rgb_image)

	if results.multi_hand_landmarks:
	hand_landmarks = results.multi_hand_landmarks[0]

	mp_drawing.draw_landmarks(
	image, hand_landmarks, mp_hands.HAND_CONNECTIONS,
	mp_drawing.DrawingSpec(color=(0, 0, 255), thickness=2, circle_radius=2),
	mp_drawing.DrawingSpec(color=(0, 255, 0), thickness=2)
	)

	features = extract_hand_features(hand_landmarks)
	gesture, confidence = predict_gesture(features)

	if confidence > 0.5:
	self.gesture_history.append(gesture)
	self.gesture_history = self.gesture_history[-10:] # Keep last 10 gestures

	with prediction_lock:
	current_prediction["gesture"] = gesture
	current_prediction["confidence"] = confidence

	# Update interpretation periodically
	if current_time - self.last_interpretation_time > self.interpretation_interval:
	self.last_interpretation_time = current_time
	interpretation = get_gesture_interpretation(self.gesture_history)
	if interpretation:
	current_prediction["interpretation"] = interpretation
	else:
	with prediction_lock:
	current_prediction["gesture"] = "No hand detected"
	current_prediction["confidence"] = 0.0
	except Exception as e:
	print(f"Processing error: {e}")
	with prediction_lock:
	current_prediction["gesture"] = "Processing error"
	current_prediction["confidence"] = 0.0

	# Add text overlay
	with prediction_lock:
	display_text = current_prediction["gesture"]
	confidence = current_prediction["confidence"]

	color = (0, 255, 0) if confidence > 0.5 else (0, 0, 255)
	cv2.putText(
	image,
	f"{display_text} ({confidence:.2f})",
	(20, 40),
	cv2.FONT_HERSHEY_SIMPLEX,
	1.0,
	color,
	2
	)

	return VideoFrame.from_ndarray(image, format="bgr24")

	def main():
	st.title("🤟 AI Sign Language Interpreter")
	st.markdown("Transform sign language into text with real-time AI interpretation")

	# Create columns for layout
	col1, col2 = st.columns([2, 1])

	with col1:
	st.markdown("### 📹 Live Camera Feed")
	ctx = webrtc_streamer(
	key="gesture_recognition",
	video_processor_factory=GestureRecognizer,
	rtc_configuration=RTC_CONFIGURATION,
	media_stream_constraints={"video": True, "audio": False},
	async_processing=True,
	)

	with col2:
	st.markdown("### 🎯 Recognition Results")

	# Create placeholders for dynamic updates
	prediction_placeholder = st.empty()
	confidence_bar = st.empty()
	interpretation_placeholder = st.empty()

	if ctx.state.playing:
	while ctx.state.playing:
	with prediction_lock:
	gesture = current_prediction["gesture"]
	confidence = current_prediction["confidence"]
	interpretation = current_prediction["interpretation"]

	# Display current gesture
	with prediction_placeholder.container():
	st.markdown("""
	<div class="prediction-box">
	<h3 style="font-size: 1.5rem; margin-bottom: 0.5rem;">Current Gesture</h3>
	<p style="font-size: 2rem; font-weight: bold; color: #4c51bf;">{}</p>
	</div>
	""".format(gesture), unsafe_allow_html=True)

	# Display confidence bar
	if confidence > 0.5 and gesture not in ["No hand detected", "Processing error", "Error"]:
	confidence_bar.progress(confidence)

	# Display interpretation
	if interpretation:
	with interpretation_placeholder.container():
	st.markdown("""
	<div class="interpretation-box">
	<h3 style="font-size: 1.5rem; margin-bottom: 0.5rem;">AI Interpretation</h3>
	<p style="font-size: 1.2rem;">{}</p>
	</div>
	""".format(interpretation), unsafe_allow_html=True)
	else:
	confidence_bar.progress(0)
	if gesture == "No hand detected":
	interpretation_placeholder.info("Show your hand to begin")
	else:
	interpretation_placeholder.warning("Gesture not recognized clearly")

	time.sleep(0.1)
	else:
	st.warning("📷 Turn on your webcam to start!")

	# Instructions and supported gestures
	st.markdown("---")

	# Display supported gestures in an organized grid
	st.markdown("### 🎯 Supported Gestures")

	# Numbers
	st.markdown("#### 🔢 Numbers")
	number_labels = [label for label in labels if label.isdigit()]
	if number_labels:
	cols = st.columns(len(number_labels))
	for i, label in enumerate(number_labels):
	with cols[i]:
	st.markdown(f"""
	<div style="text-align: center; padding: 1rem; background-color: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1);">
	<h3 style="font-size: 1.5rem; color: #4c51bf;">{label}</h3>
	</div>
	""", unsafe_allow_html=True)

	# Letters
	st.markdown("#### 🔤 Letters")
	letter_labels = [label for label in labels if label.isalpha()]
	if letter_labels:
	# Display letters in rows of 8
	for i in range(0, len(letter_labels), 8):
	row_labels = letter_labels[i:i+8]
	cols = st.columns(len(row_labels))
	for j, label in enumerate(row_labels):
	with cols[j]:
	st.markdown(f"""
	<div style="text-align: center; padding: 1rem; background-color: white; border-radius: 8px; box-shadow: 0 2px 4px rgba(0,0,0,0.1);">
	<h3 style="font-size: 1.5rem; color: #4c51bf;">{label}</h3>
	</div>
	""", unsafe_allow_html=True)

	if __name__ == "__main__":
	main()