Anushree1 commited on
Commit
7ab493c
·
verified ·
1 Parent(s): e618615

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +76 -59
app.py CHANGED
@@ -1,66 +1,83 @@
1
  import cv2
2
  import mediapipe as mp
 
 
 
3
 
4
- # Initialize MediaPipe Hand module
 
5
  mp_hands = mp.solutions.hands
6
- mp_drawing = mp.solutions.drawing_utils
7
 
8
- # Initialize hand detector
 
 
 
9
  hands = mp_hands.Hands(min_detection_confidence=0.7, min_tracking_confidence=0.7)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
 
11
- # Initialize webcam feed
12
- cap = cv2.VideoCapture(1)
13
-
14
- # Check if the camera opened successfully
15
- if not cap.isOpened():
16
- print("Error: Could not open webcam.")
17
- exit()
18
-
19
- # Define a simple gesture control function (e.g., if a gesture is detected, print a command)
20
- def gesture_control(hand_landmarks):
21
- try:
22
- # Example: Detecting 'thumb up' gesture (simplified for demonstration)
23
- thumb_tip = hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP]
24
- index_tip = hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP]
25
-
26
- # Thumb up gesture detection based on relative positions of thumb and index
27
- if thumb_tip.y < index_tip.y: # Thumb above the index finger
28
- print("Thumb Up Gesture Detected: Activating Gesture Control")
29
- except IndexError:
30
- print("Error: Invalid landmarks detected.")
31
-
32
- # Start video capture loop
33
- while True:
34
- ret, frame = cap.read()
35
- if not ret:
36
- print("Error: Failed to capture frame.")
37
- break
38
-
39
- # Flip the frame horizontally for a selfie-view display
40
- frame = cv2.flip(frame, 1)
41
-
42
- # Convert the image to RGB (MediaPipe uses RGB format)
43
- rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
44
-
45
- # Process the frame and get hand landmarks
46
- hand_results = hands.process(rgb_frame)
47
-
48
- # If hand landmarks are detected
49
- if hand_results.multi_hand_landmarks:
50
- for hand_landmarks in hand_results.multi_hand_landmarks:
51
- # Draw hand landmarks on the frame
52
- mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
53
-
54
- # Call the gesture control function
55
- gesture_control(hand_landmarks)
56
-
57
- # Display the frame with hand landmarks
58
- cv2.imshow("Hand Gesture Control", frame)
59
-
60
- # Break the loop on 'q' key press
61
- if cv2.waitKey(1) & 0xFF == ord('q'):
62
- break
63
-
64
- # Release the video capture and close windows
65
- cap.release()
66
- cv2.destroyAllWindows()
 
1
  import cv2
2
  import mediapipe as mp
3
+ import tensorflow as tf
4
+ import numpy as np
5
+ import gradio as gr
6
 
7
+ # MediaPipe setup for hand and face landmark detection
8
+ mp_face_mesh = mp.solutions.face_mesh
9
  mp_hands = mp.solutions.hands
 
10
 
11
+ # Load the pre-trained sign language model
12
+ model = tf.keras.models.load_model('sign_language_model.h5')
13
+
14
+ # Initialize MediaPipe Hands and FaceMesh
15
  hands = mp_hands.Hands(min_detection_confidence=0.7, min_tracking_confidence=0.7)
16
+ face_mesh = mp_face_mesh.FaceMesh(min_detection_confidence=0.7, min_tracking_confidence=0.7)
17
+
18
+ # Function for webcam input and processing landmarks
19
+ def process_frame(frame):
20
+ # Convert the frame to RGB (as MediaPipe uses RGB images)
21
+ image_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
22
+ results_hands = hands.process(image_rgb)
23
+ results_face = face_mesh.process(image_rgb)
24
+
25
+ # Draw landmarks for hands
26
+ if results_hands.multi_hand_landmarks:
27
+ for hand_landmarks in results_hands.multi_hand_landmarks:
28
+ mp.solutions.drawing_utils.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
29
+
30
+ # Draw landmarks for face
31
+ if results_face.multi_face_landmarks:
32
+ for face_landmarks in results_face.multi_face_landmarks:
33
+ mp.solutions.drawing_utils.draw_landmarks(frame, face_landmarks, mp_face_mesh.FACEMESH_CONTOURS)
34
+
35
+ # Convert back to BGR for OpenCV
36
+ return frame
37
+
38
+ # Function for performing sign language gesture prediction
39
+ def predict_gesture(frame):
40
+ # Preprocessing the frame for gesture recognition
41
+ image = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
42
+ image = cv2.resize(image, (28, 28))
43
+ image = np.expand_dims(image, axis=-1)
44
+ image = image / 255.0
45
+ image = np.expand_dims(image, axis=0)
46
+
47
+ # Prediction using the loaded model
48
+ prediction = model.predict(image)
49
+ predicted_class = np.argmax(prediction, axis=1)
50
+
51
+ # Mapping the predicted class to corresponding ASL alphabet
52
+ asl_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
53
+ return asl_alphabet[predicted_class[0]]
54
+
55
+ # Gradio Interface for live webcam input and prediction
56
+ def webcam_input():
57
+ cap = cv2.VideoCapture(0) # Open the webcam (try 1 or 2 if 0 doesn't work)
58
+ if not cap.isOpened():
59
+ return "Error: Cannot open camera"
60
+
61
+ while True:
62
+ ret, frame = cap.read()
63
+ if not ret:
64
+ break
65
+
66
+ # Process the frame to detect landmarks and make predictions
67
+ processed_frame = process_frame(frame)
68
+ prediction = predict_gesture(processed_frame)
69
+
70
+ # Display predicted gesture on the frame
71
+ cv2.putText(processed_frame, f"Prediction: {prediction}", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
72
+
73
+ # Show the frame with predictions
74
+ cv2.imshow("Hand and Facial Landmark Recognition", processed_frame)
75
+
76
+ if cv2.waitKey(1) & 0xFF == ord('q'):
77
+ break
78
+
79
+ cap.release()
80
+ cv2.destroyAllWindows()
81
 
82
+ # Start the Gradio interface
83
+ gr.Interface(fn=webcam_input, live=True, capture_video=True).launch()