Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,66 +1,83 @@
|
|
| 1 |
import cv2
|
| 2 |
import mediapipe as mp
|
|
|
|
|
|
|
|
|
|
| 3 |
|
| 4 |
-
#
|
|
|
|
| 5 |
mp_hands = mp.solutions.hands
|
| 6 |
-
mp_drawing = mp.solutions.drawing_utils
|
| 7 |
|
| 8 |
-
#
|
|
|
|
|
|
|
|
|
|
| 9 |
hands = mp_hands.Hands(min_detection_confidence=0.7, min_tracking_confidence=0.7)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
-
#
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
# Check if the camera opened successfully
|
| 15 |
-
if not cap.isOpened():
|
| 16 |
-
print("Error: Could not open webcam.")
|
| 17 |
-
exit()
|
| 18 |
-
|
| 19 |
-
# Define a simple gesture control function (e.g., if a gesture is detected, print a command)
|
| 20 |
-
def gesture_control(hand_landmarks):
|
| 21 |
-
try:
|
| 22 |
-
# Example: Detecting 'thumb up' gesture (simplified for demonstration)
|
| 23 |
-
thumb_tip = hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP]
|
| 24 |
-
index_tip = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
|
| 25 |
-
|
| 26 |
-
# Thumb up gesture detection based on relative positions of thumb and index
|
| 27 |
-
if thumb_tip.y < index_tip.y: # Thumb above the index finger
|
| 28 |
-
print("Thumb Up Gesture Detected: Activating Gesture Control")
|
| 29 |
-
except IndexError:
|
| 30 |
-
print("Error: Invalid landmarks detected.")
|
| 31 |
-
|
| 32 |
-
# Start video capture loop
|
| 33 |
-
while True:
|
| 34 |
-
ret, frame = cap.read()
|
| 35 |
-
if not ret:
|
| 36 |
-
print("Error: Failed to capture frame.")
|
| 37 |
-
break
|
| 38 |
-
|
| 39 |
-
# Flip the frame horizontally for a selfie-view display
|
| 40 |
-
frame = cv2.flip(frame, 1)
|
| 41 |
-
|
| 42 |
-
# Convert the image to RGB (MediaPipe uses RGB format)
|
| 43 |
-
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 44 |
-
|
| 45 |
-
# Process the frame and get hand landmarks
|
| 46 |
-
hand_results = hands.process(rgb_frame)
|
| 47 |
-
|
| 48 |
-
# If hand landmarks are detected
|
| 49 |
-
if hand_results.multi_hand_landmarks:
|
| 50 |
-
for hand_landmarks in hand_results.multi_hand_landmarks:
|
| 51 |
-
# Draw hand landmarks on the frame
|
| 52 |
-
mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
| 53 |
-
|
| 54 |
-
# Call the gesture control function
|
| 55 |
-
gesture_control(hand_landmarks)
|
| 56 |
-
|
| 57 |
-
# Display the frame with hand landmarks
|
| 58 |
-
cv2.imshow("Hand Gesture Control", frame)
|
| 59 |
-
|
| 60 |
-
# Break the loop on 'q' key press
|
| 61 |
-
if cv2.waitKey(1) & 0xFF == ord('q'):
|
| 62 |
-
break
|
| 63 |
-
|
| 64 |
-
# Release the video capture and close windows
|
| 65 |
-
cap.release()
|
| 66 |
-
cv2.destroyAllWindows()
|
|
|
|
| 1 |
import cv2
|
| 2 |
import mediapipe as mp
|
| 3 |
+
import tensorflow as tf
|
| 4 |
+
import numpy as np
|
| 5 |
+
import gradio as gr
|
| 6 |
|
| 7 |
+
# MediaPipe setup for hand and face landmark detection
|
| 8 |
+
mp_face_mesh = mp.solutions.face_mesh
|
| 9 |
mp_hands = mp.solutions.hands
|
|
|
|
| 10 |
|
| 11 |
+
# Load the pre-trained sign language model
|
| 12 |
+
model = tf.keras.models.load_model('sign_language_model.h5')
|
| 13 |
+
|
| 14 |
+
# Initialize MediaPipe Hands and FaceMesh
|
| 15 |
hands = mp_hands.Hands(min_detection_confidence=0.7, min_tracking_confidence=0.7)
|
| 16 |
+
face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.7, min_tracking_confidence=0.7)
|
| 17 |
+
|
| 18 |
+
# Function for webcam input and processing landmarks
|
| 19 |
+
def process_frame(frame):
|
| 20 |
+
# Convert the frame to RGB (as MediaPipe uses RGB images)
|
| 21 |
+
image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
| 22 |
+
results_hands = hands.process(image_rgb)
|
| 23 |
+
results_face = face_mesh.process(image_rgb)
|
| 24 |
+
|
| 25 |
+
# Draw landmarks for hands
|
| 26 |
+
if results_hands.multi_hand_landmarks:
|
| 27 |
+
for hand_landmarks in results_hands.multi_hand_landmarks:
|
| 28 |
+
mp.solutions.drawing_utils.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
|
| 29 |
+
|
| 30 |
+
# Draw landmarks for face
|
| 31 |
+
if results_face.multi_face_landmarks:
|
| 32 |
+
for face_landmarks in results_face.multi_face_landmarks:
|
| 33 |
+
mp.solutions.drawing_utils.draw_landmarks(frame, face_landmarks, mp_face_mesh.FACEMESH_CONTOURS)
|
| 34 |
+
|
| 35 |
+
# Convert back to BGR for OpenCV
|
| 36 |
+
return frame
|
| 37 |
+
|
| 38 |
+
# Function for performing sign language gesture prediction
|
| 39 |
+
def predict_gesture(frame):
|
| 40 |
+
# Preprocessing the frame for gesture recognition
|
| 41 |
+
image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
| 42 |
+
image = cv2.resize(image, (28, 28))
|
| 43 |
+
image = np.expand_dims(image, axis=-1)
|
| 44 |
+
image = image / 255.0
|
| 45 |
+
image = np.expand_dims(image, axis=0)
|
| 46 |
+
|
| 47 |
+
# Prediction using the loaded model
|
| 48 |
+
prediction = model.predict(image)
|
| 49 |
+
predicted_class = np.argmax(prediction, axis=1)
|
| 50 |
+
|
| 51 |
+
# Mapping the predicted class to corresponding ASL alphabet
|
| 52 |
+
asl_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
| 53 |
+
return asl_alphabet[predicted_class[0]]
|
| 54 |
+
|
| 55 |
+
# Gradio Interface for live webcam input and prediction
|
| 56 |
+
def webcam_input():
|
| 57 |
+
cap = cv2.VideoCapture(0) # Open the webcam (try 1 or 2 if 0 doesn't work)
|
| 58 |
+
if not cap.isOpened():
|
| 59 |
+
return "Error: Cannot open camera"
|
| 60 |
+
|
| 61 |
+
while True:
|
| 62 |
+
ret, frame = cap.read()
|
| 63 |
+
if not ret:
|
| 64 |
+
break
|
| 65 |
+
|
| 66 |
+
# Process the frame to detect landmarks and make predictions
|
| 67 |
+
processed_frame = process_frame(frame)
|
| 68 |
+
prediction = predict_gesture(processed_frame)
|
| 69 |
+
|
| 70 |
+
# Display predicted gesture on the frame
|
| 71 |
+
cv2.putText(processed_frame, f"Prediction: {prediction}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
|
| 72 |
+
|
| 73 |
+
# Show the frame with predictions
|
| 74 |
+
cv2.imshow("Hand and Facial Landmark Recognition", processed_frame)
|
| 75 |
+
|
| 76 |
+
if cv2.waitKey(1) & 0xFF == ord('q'):
|
| 77 |
+
break
|
| 78 |
+
|
| 79 |
+
cap.release()
|
| 80 |
+
cv2.destroyAllWindows()
|
| 81 |
|
| 82 |
+
# Start the Gradio interface
|
| 83 |
+
gr.Interface(fn=webcam_input, live=True, capture_video=True).launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|