swastikbansal commited on
Commit
7e05328
·
verified ·
1 Parent(s): 497bfdb

Implemented rest.py in app.py

Browse files
Files changed (1) hide show
  1. app.py +51 -30
app.py CHANGED
@@ -12,6 +12,13 @@ from queue import Queue, Empty
12
 
13
  import time
14
 
 
 
 
 
 
 
 
15
  app = Flask(__name__)
16
 
17
  # Queue for video frames to be displayed
@@ -87,39 +94,59 @@ utils = Utils(axes)
87
  def process_image(img):
88
  """Process a single image (np.array) to get a prediction."""
89
  global accumulated_probs, frame_count
90
-
91
  pred = None
92
  left_probs, right_probs, pose_probs = None, None, None
93
-
94
- # To improve performance, optionally mark the image as not writeable to
95
- # pass by reference.
96
  img.flags.writeable = False
97
  img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
98
  res_pose = pose.process(img_rgb)
99
  res_hands = hands.process(img_rgb)
100
  img.flags.writeable = True
 
101
 
 
102
  try:
103
- # Process hand landmarks (for both left and right hands)
104
- if res_hands.multi_hand_landmarks:
 
 
 
 
 
 
 
 
105
  for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
106
  label = handedness.classification[0].label
107
- features = utils.extract_features(hand_landmarks.landmark, res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else [])
 
 
 
108
 
109
  if label == 'Left':
110
  left_probs = left_model.predict_proba([features])[0]
111
  elif label == 'Right':
112
  right_probs = right_model.predict_proba([features])[0]
113
 
114
- # Process pose landmarks
115
- if res_pose.pose_landmarks:
 
 
 
 
116
  pose_landmarks = res_pose.pose_landmarks
117
- pose_features = utils.extract_pose_features(pose_landmarks.landmark) # Pass landmark attribute
 
118
  pose_probs = pose_model.predict_proba([pose_features])[0]
119
-
 
 
 
 
120
  except Exception as e:
121
  print(f"Feature extraction error: {e}")
122
- return {"message": "Feature extraction failed", "frame_count": frame_count}, img
123
 
124
  # Gathering all class labels used by the three models
125
  all_classes = sorted(
@@ -127,12 +154,12 @@ def process_image(img):
127
  set(right_model.classes_)).union(
128
  set(pose_model.classes_))
129
  )
130
-
131
  # Aligning probabilities with the master list of classes
132
  left_probs_aligned = np.zeros(len(all_classes))
133
  right_probs_aligned = np.zeros(len(all_classes))
134
  pose_probs_aligned = np.zeros(len(all_classes))
135
-
136
  if left_probs is not None:
137
  left_dict = dict(zip(left_model.classes_, left_probs))
138
  left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
@@ -144,13 +171,12 @@ def process_image(img):
144
  if pose_probs is not None:
145
  pose_dict = dict(zip(pose_model.classes_, pose_probs))
146
  pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
147
-
148
  # Determine the number of available sources (hand(s)/pose)
149
  num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
150
  if num_sources == 0:
151
- # Always return a tuple (dict, img) to keep the caller safe
152
- return {"message": "No valid data", "frame_count": frame_count}, img
153
-
154
  avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
155
 
156
  # Convert these averages into final percentages based on custom thresholds
@@ -160,7 +186,7 @@ def process_image(img):
160
  accumulated_probs = np.zeros_like(avg_probs)
161
  accumulated_probs += avg_probs
162
  frame_count += 1
163
-
164
  # Updating the final prediction text after a fixed no. of frames
165
  if frame_count == 5:
166
  max_idx = np.argmax(accumulated_probs)
@@ -168,8 +194,11 @@ def process_image(img):
168
  accumulated_probs = None
169
  frame_count = 0
170
 
171
- # Make the tuple return explicit (avoid precedence surprises)
172
- return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), img
 
 
 
173
 
174
  @app.route('/process_frame', methods=['POST'])
175
  def process_video_frame():
@@ -198,13 +227,7 @@ def process_video_frame():
198
  pass
199
 
200
  # Process the image to get a prediction
201
- ret = process_image(img)
202
- # Defensive unpack: process_image should return (dict, img) now, but handle older behavior gracefully
203
- if isinstance(ret, tuple) and len(ret) == 2:
204
- result, annotated_img = ret
205
- else:
206
- result = ret
207
- annotated_img = img
208
 
209
  if result.get("prediction"):
210
  # Clear the queue as we have a final prediction
@@ -234,7 +257,6 @@ def process_video_frame():
234
  }), 200
235
 
236
  except Exception as e:
237
- print("Error processing frame:", e)
238
  return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
239
 
240
 
@@ -255,7 +277,6 @@ def health_check():
255
  "models_loaded": True
256
  }), 200
257
 
258
-
259
  print("Starting prediction API server...")
260
  print("Endpoints available:")
261
  print("- POST /process_frame - Send video frame (multipart) for prediction and display")
 
12
 
13
  import time
14
 
15
+ # Rest detection parameters (tweak these for your camera/subject)
16
+ REST_SPEED_THRESHOLD :float = 30.0 # pixels/second; lower => more sensitive to rest
17
+ MIN_POINTS_FOR_REST :int = 2
18
+ REQUIRED_CONSECUTIVE_FRAMES :int = 3
19
+ REST_DELAY = 2 # Frames before considering rest
20
+
21
+
22
  app = Flask(__name__)
23
 
24
  # Queue for video frames to be displayed
 
94
  def process_image(img):
95
  """Process a single image (np.array) to get a prediction."""
96
  global accumulated_probs, frame_count
97
+
98
  pred = None
99
  left_probs, right_probs, pose_probs = None, None, None
100
+
 
 
101
  img.flags.writeable = False
102
  img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
103
  res_pose = pose.process(img_rgb)
104
  res_hands = hands.process(img_rgb)
105
  img.flags.writeable = True
106
+ annotated = img.copy()
107
 
108
+
109
  try:
110
+ resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None)
111
+ except Exception as e:
112
+ print(f"Rest detection error: {e}")
113
+ resting = False
114
+
115
+ if resting:
116
+ return {"prediction": "rest"}, annotated
117
+
118
+ try:
119
+ if getattr(res_hands, "multi_hand_landmarks", None):
120
  for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
121
  label = handedness.classification[0].label
122
+ features = utils.extract_features(
123
+ hand_landmarks.landmark,
124
+ res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else []
125
+ )
126
 
127
  if label == 'Left':
128
  left_probs = left_model.predict_proba([features])[0]
129
  elif label == 'Right':
130
  right_probs = right_model.predict_proba([features])[0]
131
 
132
+ # Draw hand landmarks on the frame
133
+ mp.solutions.drawing_utils.draw_landmarks(
134
+ img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS
135
+ )
136
+
137
+ if getattr(res_pose, "pose_landmarks", None):
138
  pose_landmarks = res_pose.pose_landmarks
139
+ pose_features = utils.extract_pose_features(pose_landmarks.landmark)
140
+ pose_prediction = pose_model.predict([pose_features])[0]
141
  pose_probs = pose_model.predict_proba([pose_features])[0]
142
+ # Draw pose landmarks on the frame
143
+ mp.solutions.drawing_utils.draw_landmarks(
144
+ img, pose_landmarks, mp_pose.POSE_CONNECTIONS
145
+ )
146
+
147
  except Exception as e:
148
  print(f"Feature extraction error: {e}")
149
+ return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated
150
 
151
  # Gathering all class labels used by the three models
152
  all_classes = sorted(
 
154
  set(right_model.classes_)).union(
155
  set(pose_model.classes_))
156
  )
157
+
158
  # Aligning probabilities with the master list of classes
159
  left_probs_aligned = np.zeros(len(all_classes))
160
  right_probs_aligned = np.zeros(len(all_classes))
161
  pose_probs_aligned = np.zeros(len(all_classes))
162
+
163
  if left_probs is not None:
164
  left_dict = dict(zip(left_model.classes_, left_probs))
165
  left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
 
171
  if pose_probs is not None:
172
  pose_dict = dict(zip(pose_model.classes_, pose_probs))
173
  pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
174
+
175
  # Determine the number of available sources (hand(s)/pose)
176
  num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
177
  if num_sources == 0:
178
+ return {"message": "No valid data", "frame_count": frame_count}, annotated
179
+
 
180
  avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
181
 
182
  # Convert these averages into final percentages based on custom thresholds
 
186
  accumulated_probs = np.zeros_like(avg_probs)
187
  accumulated_probs += avg_probs
188
  frame_count += 1
189
+
190
  # Updating the final prediction text after a fixed no. of frames
191
  if frame_count == 5:
192
  max_idx = np.argmax(accumulated_probs)
 
194
  accumulated_probs = None
195
  frame_count = 0
196
 
197
+ return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated
198
+
199
+ @app.route('/')
200
+ def home():
201
+ return "<h1>Signify API</h1>"
202
 
203
  @app.route('/process_frame', methods=['POST'])
204
  def process_video_frame():
 
227
  pass
228
 
229
  # Process the image to get a prediction
230
+ result, annotated_img = process_image(img)
 
 
 
 
 
 
231
 
232
  if result.get("prediction"):
233
  # Clear the queue as we have a final prediction
 
257
  }), 200
258
 
259
  except Exception as e:
 
260
  return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
261
 
262
 
 
277
  "models_loaded": True
278
  }), 200
279
 
 
280
  print("Starting prediction API server...")
281
  print("Endpoints available:")
282
  print("- POST /process_frame - Send video frame (multipart) for prediction and display")