swastikbansal commited on
Commit
cd1e957
·
1 Parent(s): b8dfa86

Updated main.py file name to app,py

Browse files
Files changed (2) hide show
  1. .gitignore +1 -0
  2. main.py → app.py +262 -262
.gitignore ADDED
@@ -0,0 +1 @@
 
 
1
+ .venv
main.py → app.py RENAMED
@@ -1,262 +1,262 @@
1
- import warnings
2
- warnings.filterwarnings("ignore")
3
-
4
- from flask import Flask, request, jsonify
5
- import pickle
6
- import numpy as np
7
- from utils import Utils
8
- from matplotlib import pyplot as plt
9
- import mediapipe as mp
10
- import cv2
11
- from queue import Queue
12
-
13
- import time
14
-
15
- app = Flask(__name__)
16
-
17
- # Queue for video frames to be displayed
18
- frame_queue = Queue(maxsize=10)
19
-
20
- # Global variables for frame accumulation
21
- accumulated_probs = None
22
- frame_count = 0
23
-
24
-
25
- def calulating_percentage(avg, all_classes):
26
- """
27
- Given a list of average probabilities (avg) and corresponding class labels (all_classes),
28
- returns a list of percentage values adjusted by class-specific thresholds.
29
- """
30
-
31
- individual_threshold = {
32
- "sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9,
33
- "paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9,
34
- "accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9,
35
- "poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9,
36
- "important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9,
37
- "pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9,
38
- "school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9,
39
- "slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9,
40
- "poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9
41
- }
42
-
43
- threshold_percentage = []
44
- for score, cls in zip(avg, all_classes):
45
- threshold_val = individual_threshold.get(cls.lower(), 1.0)
46
- threshold_percentage.append(score * 100 / threshold_val)
47
-
48
- return threshold_percentage
49
-
50
- def load_models():
51
- """Load all models"""
52
- left_model_filename = r'Models\left_model.p'
53
- right_model_filename = r'Models\right_model.p'
54
- pose_model_filename = r'Models\pose_model.p'
55
-
56
- def load_model(filename):
57
- with open(filename, 'rb') as f:
58
- model_data = pickle.load(f)
59
- return model_data['model']
60
-
61
- return (load_model(left_model_filename),
62
- load_model(right_model_filename),
63
- load_model(pose_model_filename))
64
-
65
- # Load models once at startup
66
- left_model, right_model, pose_model = load_models()
67
-
68
- # Initialize MediaPipe solutions
69
- mp_hands = mp.solutions.hands
70
- mp_pose = mp.solutions.pose
71
- hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5)
72
- pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7)
73
-
74
- # Initialize Utils with axes for palm orientation
75
- axes = {
76
- "x": np.array([1, 0, 0]),
77
- "-x": np.array([-1, 0, 0]),
78
- "y": np.array([0, 1, 0]),
79
- "-y": np.array([0, -1, 0]),
80
- "z": np.array([0, 0, 1]),
81
- "-z": np.array([0, 0, -1]),
82
- }
83
-
84
- utils = Utils(axes)
85
-
86
-
87
- def process_image(img):
88
- """Process a single image (np.array) to get a prediction."""
89
- global accumulated_probs, frame_count
90
-
91
- pred = None
92
- left_probs, right_probs, pose_probs = None, None, None
93
-
94
- # To improve performance, optionally mark the image as not writeable to
95
- # pass by reference.
96
- img.flags.writeable = False
97
- img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
98
- res_pose = pose.process(img_rgb)
99
- res_hands = hands.process(img_rgb)
100
- img.flags.writeable = True
101
-
102
- try:
103
- # Process hand landmarks (for both left and right hands)
104
- if res_hands.multi_hand_landmarks:
105
- for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
106
- label = handedness.classification[0].label
107
- features = utils.extract_features(hand_landmarks.landmark, res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else [])
108
-
109
- if label == 'Left':
110
- left_prediction = left_model.predict([features])[0]
111
- left_probs = left_model.predict_proba([features])[0]
112
- elif label == 'Right':
113
- right_prediction = right_model.predict([features])[0]
114
- right_probs = right_model.predict_proba([features])[0]
115
-
116
- # Process pose landmarks
117
- if res_pose.pose_landmarks:
118
- pose_landmarks = res_pose.pose_landmarks
119
- pose_features = utils.extract_pose_features(pose_landmarks.landmark) # Pass landmark attribute
120
- pose_prediction = pose_model.predict([pose_features])[0]
121
- pose_probs = pose_model.predict_proba([pose_features])[0]
122
-
123
- except Exception as e:
124
- print(f"Feature extraction error: {e}")
125
- return {"message": "Feature extraction failed", "frame_count": frame_count}, img
126
-
127
- # Gathering all class labels used by the three models
128
- all_classes = sorted(
129
- set(left_model.classes_).union(
130
- set(right_model.classes_)).union(
131
- set(pose_model.classes_))
132
- )
133
-
134
- # Aligning probabilities with the master list of classes
135
- left_probs_aligned = np.zeros(len(all_classes))
136
- right_probs_aligned = np.zeros(len(all_classes))
137
- pose_probs_aligned = np.zeros(len(all_classes))
138
-
139
- if left_probs is not None:
140
- left_dict = dict(zip(left_model.classes_, left_probs))
141
- left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
142
-
143
- if right_probs is not None:
144
- right_dict = dict(zip(right_model.classes_, right_probs))
145
- right_probs_aligned = np.array([right_dict.get(cls, 0) for cls in all_classes]) * 100
146
-
147
- if pose_probs is not None:
148
- pose_dict = dict(zip(pose_model.classes_, pose_probs))
149
- pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
150
-
151
- # Determine the number of available sources (hand(s)/pose)
152
- num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
153
- if num_sources == 0:
154
- return {"message": "No valid data", "frame_count": frame_count}
155
-
156
- avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
157
-
158
- # Convert these averages into final percentages based on custom thresholds
159
- avg_probs = calulating_percentage(avg, all_classes)
160
-
161
- if accumulated_probs is None:
162
- accumulated_probs = np.zeros_like(avg_probs)
163
- accumulated_probs += avg_probs
164
- frame_count += 1
165
-
166
- # Updating the final prediction text after a fixed no. of frames
167
- if frame_count == 5:
168
- max_idx = np.argmax(accumulated_probs)
169
- pred = all_classes[max_idx]
170
- accumulated_probs = None
171
- frame_count = 0
172
-
173
- return {"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}, img
174
-
175
- @app.route('/process_frame', methods=['POST'])
176
- def process_video_frame():
177
- """Receive a video frame, display it, and return a prediction."""
178
- try:
179
- if 'frame' not in request.files:
180
- return jsonify({"status": "error", "message": "No frame file received"}), 400
181
-
182
- file = request.files['frame']
183
- img_data = file.read()
184
-
185
- np_arr = np.frombuffer(img_data, np.uint8)
186
- img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
187
-
188
- if img is None:
189
- return jsonify({"status": "error", "message": "Failed to decode image"}), 400
190
-
191
- # Rotate the frame 90 degrees clockwise
192
- img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
193
-
194
- if frame_queue.full():
195
- try:
196
- # Non-blocking get to remove the oldest frame
197
- frame_queue.get_nowait()
198
- except Queue.Empty:
199
- pass
200
-
201
- # Process the image to get a prediction
202
- result, annotated_img = process_image(img)
203
-
204
- if result.get("prediction"):
205
- # Clear the queue as we have a final prediction
206
- while not frame_queue.empty():
207
- try:
208
- frame_queue.get_nowait()
209
- except Queue.Empty:
210
- break
211
-
212
- text = f"Prediction: {result['prediction']}"
213
- cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
214
- frame_queue.put(annotated_img)
215
-
216
- return jsonify({
217
- "status": "success",
218
- "prediction": result["prediction"]
219
- }), 200
220
- else:
221
- text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)"
222
- cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
223
- frame_queue.put(annotated_img)
224
-
225
- return jsonify({
226
- "status": "collecting",
227
- "message": result.get("message", "Processing frames"),
228
- "frame_count": result.get("frame_count", 0)
229
- }), 200
230
-
231
- except Exception as e:
232
- return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
233
-
234
-
235
- @app.route('/reset', methods=['POST'])
236
- def reset_accumulation():
237
- """Reset the frame accumulation"""
238
- global accumulated_probs, frame_count
239
- accumulated_probs = None
240
- frame_count = 0
241
- return jsonify({"status": "success", "message": "Accumulation reset"}), 200
242
-
243
- @app.route('/health', methods=['GET'])
244
- def health_check():
245
- """Health check endpoint"""
246
- return jsonify({
247
- "status": "healthy",
248
- "frame_count": frame_count,
249
- "models_loaded": True
250
- }), 200
251
-
252
- if __name__ == '__main__':
253
- print("Starting prediction API server...")
254
- print("Endpoints available:")
255
- print("- POST /process_frame - Send video frame (multipart) for prediction and display")
256
- print("- POST /reset - Reset frame accumulation")
257
- print("- GET /health - Health check")
258
- print("Press 'q' in the 'Live Stream' window to quit.")
259
-
260
- # use_reloader=False is important for threads
261
- app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)
262
-
 
1
+ import warnings
2
+ warnings.filterwarnings("ignore")
3
+
4
+ from flask import Flask, request, jsonify
5
+ import pickle
6
+ import numpy as np
7
+ from utils import Utils
8
+ from matplotlib import pyplot as plt
9
+ import mediapipe as mp
10
+ import cv2
11
+ from queue import Queue
12
+
13
+ import time
14
+
15
+ app = Flask(__name__)
16
+
17
+ # Queue for video frames to be displayed
18
+ frame_queue = Queue(maxsize=10)
19
+
20
+ # Global variables for frame accumulation
21
+ accumulated_probs = None
22
+ frame_count = 0
23
+
24
+
25
+ def calulating_percentage(avg, all_classes):
26
+ """
27
+ Given a list of average probabilities (avg) and corresponding class labels (all_classes),
28
+ returns a list of percentage values adjusted by class-specific thresholds.
29
+ """
30
+
31
+ individual_threshold = {
32
+ "sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9,
33
+ "paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9,
34
+ "accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9,
35
+ "poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9,
36
+ "important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9,
37
+ "pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9,
38
+ "school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9,
39
+ "slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9,
40
+ "poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9
41
+ }
42
+
43
+ threshold_percentage = []
44
+ for score, cls in zip(avg, all_classes):
45
+ threshold_val = individual_threshold.get(cls.lower(), 1.0)
46
+ threshold_percentage.append(score * 100 / threshold_val)
47
+
48
+ return threshold_percentage
49
+
50
+ def load_models():
51
+ """Load all models"""
52
+ left_model_filename = r'Models\left_model.p'
53
+ right_model_filename = r'Models\right_model.p'
54
+ pose_model_filename = r'Models\pose_model.p'
55
+
56
+ def load_model(filename):
57
+ with open(filename, 'rb') as f:
58
+ model_data = pickle.load(f)
59
+ return model_data['model']
60
+
61
+ return (load_model(left_model_filename),
62
+ load_model(right_model_filename),
63
+ load_model(pose_model_filename))
64
+
65
+ # Load models once at startup
66
+ left_model, right_model, pose_model = load_models()
67
+
68
+ # Initialize MediaPipe solutions
69
+ mp_hands = mp.solutions.hands
70
+ mp_pose = mp.solutions.pose
71
+ hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5)
72
+ pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7)
73
+
74
+ # Initialize Utils with axes for palm orientation
75
+ axes = {
76
+ "x": np.array([1, 0, 0]),
77
+ "-x": np.array([-1, 0, 0]),
78
+ "y": np.array([0, 1, 0]),
79
+ "-y": np.array([0, -1, 0]),
80
+ "z": np.array([0, 0, 1]),
81
+ "-z": np.array([0, 0, -1]),
82
+ }
83
+
84
+ utils = Utils(axes)
85
+
86
+
87
+ def process_image(img):
88
+ """Process a single image (np.array) to get a prediction."""
89
+ global accumulated_probs, frame_count
90
+
91
+ pred = None
92
+ left_probs, right_probs, pose_probs = None, None, None
93
+
94
+ # To improve performance, optionally mark the image as not writeable to
95
+ # pass by reference.
96
+ img.flags.writeable = False
97
+ img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
98
+ res_pose = pose.process(img_rgb)
99
+ res_hands = hands.process(img_rgb)
100
+ img.flags.writeable = True
101
+
102
+ try:
103
+ # Process hand landmarks (for both left and right hands)
104
+ if res_hands.multi_hand_landmarks:
105
+ for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
106
+ label = handedness.classification[0].label
107
+ features = utils.extract_features(hand_landmarks.landmark, res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else [])
108
+
109
+ if label == 'Left':
110
+ left_prediction = left_model.predict([features])[0]
111
+ left_probs = left_model.predict_proba([features])[0]
112
+ elif label == 'Right':
113
+ right_prediction = right_model.predict([features])[0]
114
+ right_probs = right_model.predict_proba([features])[0]
115
+
116
+ # Process pose landmarks
117
+ if res_pose.pose_landmarks:
118
+ pose_landmarks = res_pose.pose_landmarks
119
+ pose_features = utils.extract_pose_features(pose_landmarks.landmark) # Pass landmark attribute
120
+ pose_prediction = pose_model.predict([pose_features])[0]
121
+ pose_probs = pose_model.predict_proba([pose_features])[0]
122
+
123
+ except Exception as e:
124
+ print(f"Feature extraction error: {e}")
125
+ return {"message": "Feature extraction failed", "frame_count": frame_count}, img
126
+
127
+ # Gathering all class labels used by the three models
128
+ all_classes = sorted(
129
+ set(left_model.classes_).union(
130
+ set(right_model.classes_)).union(
131
+ set(pose_model.classes_))
132
+ )
133
+
134
+ # Aligning probabilities with the master list of classes
135
+ left_probs_aligned = np.zeros(len(all_classes))
136
+ right_probs_aligned = np.zeros(len(all_classes))
137
+ pose_probs_aligned = np.zeros(len(all_classes))
138
+
139
+ if left_probs is not None:
140
+ left_dict = dict(zip(left_model.classes_, left_probs))
141
+ left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
142
+
143
+ if right_probs is not None:
144
+ right_dict = dict(zip(right_model.classes_, right_probs))
145
+ right_probs_aligned = np.array([right_dict.get(cls, 0) for cls in all_classes]) * 100
146
+
147
+ if pose_probs is not None:
148
+ pose_dict = dict(zip(pose_model.classes_, pose_probs))
149
+ pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
150
+
151
+ # Determine the number of available sources (hand(s)/pose)
152
+ num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
153
+ if num_sources == 0:
154
+ return {"message": "No valid data", "frame_count": frame_count}
155
+
156
+ avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
157
+
158
+ # Convert these averages into final percentages based on custom thresholds
159
+ avg_probs = calulating_percentage(avg, all_classes)
160
+
161
+ if accumulated_probs is None:
162
+ accumulated_probs = np.zeros_like(avg_probs)
163
+ accumulated_probs += avg_probs
164
+ frame_count += 1
165
+
166
+ # Updating the final prediction text after a fixed no. of frames
167
+ if frame_count == 5:
168
+ max_idx = np.argmax(accumulated_probs)
169
+ pred = all_classes[max_idx]
170
+ accumulated_probs = None
171
+ frame_count = 0
172
+
173
+ return {"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}, img
174
+
175
+ @app.route('/process_frame', methods=['POST'])
176
+ def process_video_frame():
177
+ """Receive a video frame, display it, and return a prediction."""
178
+ try:
179
+ if 'frame' not in request.files:
180
+ return jsonify({"status": "error", "message": "No frame file received"}), 400
181
+
182
+ file = request.files['frame']
183
+ img_data = file.read()
184
+
185
+ np_arr = np.frombuffer(img_data, np.uint8)
186
+ img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
187
+
188
+ if img is None:
189
+ return jsonify({"status": "error", "message": "Failed to decode image"}), 400
190
+
191
+ # Rotate the frame 90 degrees clockwise
192
+ img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
193
+
194
+ if frame_queue.full():
195
+ try:
196
+ # Non-blocking get to remove the oldest frame
197
+ frame_queue.get_nowait()
198
+ except Queue.Empty:
199
+ pass
200
+
201
+ # Process the image to get a prediction
202
+ result, annotated_img = process_image(img)
203
+
204
+ if result.get("prediction"):
205
+ # Clear the queue as we have a final prediction
206
+ while not frame_queue.empty():
207
+ try:
208
+ frame_queue.get_nowait()
209
+ except Queue.Empty:
210
+ break
211
+
212
+ text = f"Prediction: {result['prediction']}"
213
+ cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
214
+ frame_queue.put(annotated_img)
215
+
216
+ return jsonify({
217
+ "status": "success",
218
+ "prediction": result["prediction"]
219
+ }), 200
220
+ else:
221
+ text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)"
222
+ cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
223
+ frame_queue.put(annotated_img)
224
+
225
+ return jsonify({
226
+ "status": "collecting",
227
+ "message": result.get("message", "Processing frames"),
228
+ "frame_count": result.get("frame_count", 0)
229
+ }), 200
230
+
231
+ except Exception as e:
232
+ return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
233
+
234
+
235
+ @app.route('/reset', methods=['POST'])
236
+ def reset_accumulation():
237
+ """Reset the frame accumulation"""
238
+ global accumulated_probs, frame_count
239
+ accumulated_probs = None
240
+ frame_count = 0
241
+ return jsonify({"status": "success", "message": "Accumulation reset"}), 200
242
+
243
+ @app.route('/health', methods=['GET'])
244
+ def health_check():
245
+ """Health check endpoint"""
246
+ return jsonify({
247
+ "status": "healthy",
248
+ "frame_count": frame_count,
249
+ "models_loaded": True
250
+ }), 200
251
+
252
+ if __name__ == '__main__':
253
+ print("Starting prediction API server...")
254
+ print("Endpoints available:")
255
+ print("- POST /process_frame - Send video frame (multipart) for prediction and display")
256
+ print("- POST /reset - Reset frame accumulation")
257
+ print("- GET /health - Health check")
258
+ print("Press 'q' in the 'Live Stream' window to quit.")
259
+
260
+ # use_reloader=False is important for threads
261
+ app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)
262
+