Spaces:
Runtime error
Runtime error
Implemented rest.py in app.py
Browse files
app.py
CHANGED
|
@@ -12,6 +12,13 @@ from queue import Queue, Empty
|
|
| 12 |
|
| 13 |
import time
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
app = Flask(__name__)
|
| 16 |
|
| 17 |
# Queue for video frames to be displayed
|
|
@@ -87,39 +94,59 @@ utils = Utils(axes)
|
|
| 87 |
def process_image(img):
|
| 88 |
"""Process a single image (np.array) to get a prediction."""
|
| 89 |
global accumulated_probs, frame_count
|
| 90 |
-
|
| 91 |
pred = None
|
| 92 |
left_probs, right_probs, pose_probs = None, None, None
|
| 93 |
-
|
| 94 |
-
# To improve performance, optionally mark the image as not writeable to
|
| 95 |
-
# pass by reference.
|
| 96 |
img.flags.writeable = False
|
| 97 |
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 98 |
res_pose = pose.process(img_rgb)
|
| 99 |
res_hands = hands.process(img_rgb)
|
| 100 |
img.flags.writeable = True
|
|
|
|
| 101 |
|
|
|
|
| 102 |
try:
|
| 103 |
-
|
| 104 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
|
| 106 |
label = handedness.classification[0].label
|
| 107 |
-
features = utils.extract_features(
|
|
|
|
|
|
|
|
|
|
| 108 |
|
| 109 |
if label == 'Left':
|
| 110 |
left_probs = left_model.predict_proba([features])[0]
|
| 111 |
elif label == 'Right':
|
| 112 |
right_probs = right_model.predict_proba([features])[0]
|
| 113 |
|
| 114 |
-
|
| 115 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
pose_landmarks = res_pose.pose_landmarks
|
| 117 |
-
pose_features = utils.extract_pose_features(pose_landmarks.landmark)
|
|
|
|
| 118 |
pose_probs = pose_model.predict_proba([pose_features])[0]
|
| 119 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
except Exception as e:
|
| 121 |
print(f"Feature extraction error: {e}")
|
| 122 |
-
return {"message": "Feature extraction failed", "frame_count": frame_count},
|
| 123 |
|
| 124 |
# Gathering all class labels used by the three models
|
| 125 |
all_classes = sorted(
|
|
@@ -127,12 +154,12 @@ def process_image(img):
|
|
| 127 |
set(right_model.classes_)).union(
|
| 128 |
set(pose_model.classes_))
|
| 129 |
)
|
| 130 |
-
|
| 131 |
# Aligning probabilities with the master list of classes
|
| 132 |
left_probs_aligned = np.zeros(len(all_classes))
|
| 133 |
right_probs_aligned = np.zeros(len(all_classes))
|
| 134 |
pose_probs_aligned = np.zeros(len(all_classes))
|
| 135 |
-
|
| 136 |
if left_probs is not None:
|
| 137 |
left_dict = dict(zip(left_model.classes_, left_probs))
|
| 138 |
left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
|
|
@@ -144,13 +171,12 @@ def process_image(img):
|
|
| 144 |
if pose_probs is not None:
|
| 145 |
pose_dict = dict(zip(pose_model.classes_, pose_probs))
|
| 146 |
pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
|
| 147 |
-
|
| 148 |
# Determine the number of available sources (hand(s)/pose)
|
| 149 |
num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
|
| 150 |
if num_sources == 0:
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
|
| 155 |
|
| 156 |
# Convert these averages into final percentages based on custom thresholds
|
|
@@ -160,7 +186,7 @@ def process_image(img):
|
|
| 160 |
accumulated_probs = np.zeros_like(avg_probs)
|
| 161 |
accumulated_probs += avg_probs
|
| 162 |
frame_count += 1
|
| 163 |
-
|
| 164 |
# Updating the final prediction text after a fixed no. of frames
|
| 165 |
if frame_count == 5:
|
| 166 |
max_idx = np.argmax(accumulated_probs)
|
|
@@ -168,8 +194,11 @@ def process_image(img):
|
|
| 168 |
accumulated_probs = None
|
| 169 |
frame_count = 0
|
| 170 |
|
| 171 |
-
|
| 172 |
-
|
|
|
|
|
|
|
|
|
|
| 173 |
|
| 174 |
@app.route('/process_frame', methods=['POST'])
|
| 175 |
def process_video_frame():
|
|
@@ -198,13 +227,7 @@ def process_video_frame():
|
|
| 198 |
pass
|
| 199 |
|
| 200 |
# Process the image to get a prediction
|
| 201 |
-
|
| 202 |
-
# Defensive unpack: process_image should return (dict, img) now, but handle older behavior gracefully
|
| 203 |
-
if isinstance(ret, tuple) and len(ret) == 2:
|
| 204 |
-
result, annotated_img = ret
|
| 205 |
-
else:
|
| 206 |
-
result = ret
|
| 207 |
-
annotated_img = img
|
| 208 |
|
| 209 |
if result.get("prediction"):
|
| 210 |
# Clear the queue as we have a final prediction
|
|
@@ -234,7 +257,6 @@ def process_video_frame():
|
|
| 234 |
}), 200
|
| 235 |
|
| 236 |
except Exception as e:
|
| 237 |
-
print("Error processing frame:", e)
|
| 238 |
return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
|
| 239 |
|
| 240 |
|
|
@@ -255,7 +277,6 @@ def health_check():
|
|
| 255 |
"models_loaded": True
|
| 256 |
}), 200
|
| 257 |
|
| 258 |
-
|
| 259 |
print("Starting prediction API server...")
|
| 260 |
print("Endpoints available:")
|
| 261 |
print("- POST /process_frame - Send video frame (multipart) for prediction and display")
|
|
|
|
| 12 |
|
| 13 |
import time
|
| 14 |
|
| 15 |
+
# Rest detection parameters (tweak these for your camera/subject)
|
| 16 |
+
REST_SPEED_THRESHOLD :float = 30.0 # pixels/second; lower => more sensitive to rest
|
| 17 |
+
MIN_POINTS_FOR_REST :int = 2
|
| 18 |
+
REQUIRED_CONSECUTIVE_FRAMES :int = 3
|
| 19 |
+
REST_DELAY = 2 # Frames before considering rest
|
| 20 |
+
|
| 21 |
+
|
| 22 |
app = Flask(__name__)
|
| 23 |
|
| 24 |
# Queue for video frames to be displayed
|
|
|
|
| 94 |
def process_image(img):
|
| 95 |
"""Process a single image (np.array) to get a prediction."""
|
| 96 |
global accumulated_probs, frame_count
|
| 97 |
+
|
| 98 |
pred = None
|
| 99 |
left_probs, right_probs, pose_probs = None, None, None
|
| 100 |
+
|
|
|
|
|
|
|
| 101 |
img.flags.writeable = False
|
| 102 |
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
|
| 103 |
res_pose = pose.process(img_rgb)
|
| 104 |
res_hands = hands.process(img_rgb)
|
| 105 |
img.flags.writeable = True
|
| 106 |
+
annotated = img.copy()
|
| 107 |
|
| 108 |
+
|
| 109 |
try:
|
| 110 |
+
resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None)
|
| 111 |
+
except Exception as e:
|
| 112 |
+
print(f"Rest detection error: {e}")
|
| 113 |
+
resting = False
|
| 114 |
+
|
| 115 |
+
if resting:
|
| 116 |
+
return {"prediction": "rest"}, annotated
|
| 117 |
+
|
| 118 |
+
try:
|
| 119 |
+
if getattr(res_hands, "multi_hand_landmarks", None):
|
| 120 |
for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
|
| 121 |
label = handedness.classification[0].label
|
| 122 |
+
features = utils.extract_features(
|
| 123 |
+
hand_landmarks.landmark,
|
| 124 |
+
res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else []
|
| 125 |
+
)
|
| 126 |
|
| 127 |
if label == 'Left':
|
| 128 |
left_probs = left_model.predict_proba([features])[0]
|
| 129 |
elif label == 'Right':
|
| 130 |
right_probs = right_model.predict_proba([features])[0]
|
| 131 |
|
| 132 |
+
# Draw hand landmarks on the frame
|
| 133 |
+
mp.solutions.drawing_utils.draw_landmarks(
|
| 134 |
+
img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
if getattr(res_pose, "pose_landmarks", None):
|
| 138 |
pose_landmarks = res_pose.pose_landmarks
|
| 139 |
+
pose_features = utils.extract_pose_features(pose_landmarks.landmark)
|
| 140 |
+
pose_prediction = pose_model.predict([pose_features])[0]
|
| 141 |
pose_probs = pose_model.predict_proba([pose_features])[0]
|
| 142 |
+
# Draw pose landmarks on the frame
|
| 143 |
+
mp.solutions.drawing_utils.draw_landmarks(
|
| 144 |
+
img, pose_landmarks, mp_pose.POSE_CONNECTIONS
|
| 145 |
+
)
|
| 146 |
+
|
| 147 |
except Exception as e:
|
| 148 |
print(f"Feature extraction error: {e}")
|
| 149 |
+
return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated
|
| 150 |
|
| 151 |
# Gathering all class labels used by the three models
|
| 152 |
all_classes = sorted(
|
|
|
|
| 154 |
set(right_model.classes_)).union(
|
| 155 |
set(pose_model.classes_))
|
| 156 |
)
|
| 157 |
+
|
| 158 |
# Aligning probabilities with the master list of classes
|
| 159 |
left_probs_aligned = np.zeros(len(all_classes))
|
| 160 |
right_probs_aligned = np.zeros(len(all_classes))
|
| 161 |
pose_probs_aligned = np.zeros(len(all_classes))
|
| 162 |
+
|
| 163 |
if left_probs is not None:
|
| 164 |
left_dict = dict(zip(left_model.classes_, left_probs))
|
| 165 |
left_probs_aligned = np.array([left_dict.get(cls, 0) for cls in all_classes]) * 100
|
|
|
|
| 171 |
if pose_probs is not None:
|
| 172 |
pose_dict = dict(zip(pose_model.classes_, pose_probs))
|
| 173 |
pose_probs_aligned = np.array([pose_dict.get(cls, 0) for cls in all_classes]) * 100
|
| 174 |
+
|
| 175 |
# Determine the number of available sources (hand(s)/pose)
|
| 176 |
num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
|
| 177 |
if num_sources == 0:
|
| 178 |
+
return {"message": "No valid data", "frame_count": frame_count}, annotated
|
| 179 |
+
|
|
|
|
| 180 |
avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100 * num_sources)
|
| 181 |
|
| 182 |
# Convert these averages into final percentages based on custom thresholds
|
|
|
|
| 186 |
accumulated_probs = np.zeros_like(avg_probs)
|
| 187 |
accumulated_probs += avg_probs
|
| 188 |
frame_count += 1
|
| 189 |
+
|
| 190 |
# Updating the final prediction text after a fixed no. of frames
|
| 191 |
if frame_count == 5:
|
| 192 |
max_idx = np.argmax(accumulated_probs)
|
|
|
|
| 194 |
accumulated_probs = None
|
| 195 |
frame_count = 0
|
| 196 |
|
| 197 |
+
return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated
|
| 198 |
+
|
| 199 |
+
@app.route('/')
|
| 200 |
+
def home():
|
| 201 |
+
return "<h1>Signify API</h1>"
|
| 202 |
|
| 203 |
@app.route('/process_frame', methods=['POST'])
|
| 204 |
def process_video_frame():
|
|
|
|
| 227 |
pass
|
| 228 |
|
| 229 |
# Process the image to get a prediction
|
| 230 |
+
result, annotated_img = process_image(img)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
|
| 232 |
if result.get("prediction"):
|
| 233 |
# Clear the queue as we have a final prediction
|
|
|
|
| 257 |
}), 200
|
| 258 |
|
| 259 |
except Exception as e:
|
|
|
|
| 260 |
return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
|
| 261 |
|
| 262 |
|
|
|
|
| 277 |
"models_loaded": True
|
| 278 |
}), 200
|
| 279 |
|
|
|
|
| 280 |
print("Starting prediction API server...")
|
| 281 |
print("Endpoints available:")
|
| 282 |
print("- POST /process_frame - Send video frame (multipart) for prediction and display")
|