Spaces:
Runtime error
Runtime error
File size: 13,390 Bytes
cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 416f492 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 416f492 cd1e957 cb8a780 fef9a8e cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 36ca911 cd1e957 cb8a780 cd1e957 36ca911 cd1e957 cb8a780 cd1e957 cb8a780 cd1e957 8b235ce cb8a780 8b235ce cd1e957 8b235ce f8c9b2e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 | #API
from flask import Flask, request, jsonify
# ML
import pickle
import numpy as np
from matplotlib import pyplot as plt
import mediapipe as mp
import cv2
# System modules
import threading
from queue import Queue, Empty
import warnings
import os
import warnings
warnings.filterwarnings("ignore")
#Custom modules
from utils import Utils
from CustomTrainer import Trainer
# TO DO
# Switching the model based on the response received from the model
def calulating_percentage(avg, all_classes):
"""
Given a list of average probabilities (avg) and corresponding class labels (all_classes),
returns a list of percentage values adjusted by class-specific thresholds.
"""
individual_threshold = {
"sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9,
"paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9,
"accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9,
"poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9,
"important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9,
"pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9,
"school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9,
"slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9,
"poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9
}
threshold_percentage = []
for score, cls in zip(avg, all_classes):
threshold_val = individual_threshold.get(cls.lower(), 1.0)
threshold_percentage.append(score * 100 / threshold_val)
return threshold_percentage
def load_models(model_type: str = "Default"):
"""
Load models for given model_type. Accepts values case-insensitively.
If model_type == 'custom' (case-insensitive) -> load custom models,
otherwise load default models. Raises FileNotFoundError with helpful message
if any model file is missing.
"""
model_type_normalized = (model_type or "Default").strip().lower()
if model_type_normalized == "custom":
left_model_filename = r'./Custom_Dataset/Models/left_model_new.p'
right_model_filename = r'./Custom_Dataset/Models/right_model_new.p'
pose_model_filename = r'./Custom_Dataset/Models/pose_model_new.p'
else:
left_model_filename = r'./Models/left_model.p'
right_model_filename = r'./Models/right_model.p'
pose_model_filename = r'./Models/pose_model.p'
def load_model(filename):
if not os.path.exists(filename):
raise FileNotFoundError(f"Model file not found: {filename}")
with open(filename, 'rb') as f:
model_data = pickle.load(f)
# Support both {'model': model} and plain model object pickles
if isinstance(model_data, dict) and 'model' in model_data:
return model_data['model']
return model_data
try:
return (
load_model(left_model_filename),
load_model(right_model_filename),
load_model(pose_model_filename)
)
except Exception as e:
print(f"Failed to load models for type '{model_type}': {e}")
raise
def process_image(img, USE_REST:bool = False):
"""Process a single image (np.array) to get a prediction."""
global accumulated_probs, frame_count
pred = None
left_probs, right_probs, pose_probs = None, None, None
img.flags.writeable = False
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
res_pose = pose.process(img_rgb)
res_hands = hands.process(img_rgb)
img.flags.writeable = True
annotated = img.copy()
try:
if USE_REST:
resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None)
else:
resting = False
except Exception as e:
print(f"Rest detection error: {e}")
resting = False
if resting:
return {"prediction": "rest"}, annotated
try:
if getattr(res_hands, "multi_hand_landmarks", None):
for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
label = handedness.classification[0].label
features = utils.extract_hand_features(
hand_landmarks.landmark,
res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else []
)
if label == 'Left':
left_probs = left_model.predict_proba([features])[0]
elif label == 'Right':
right_probs = right_model.predict_proba([features])[0]
# Draw hand landmarks on the frame
mp.solutions.drawing_utils.draw_landmarks(
img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS
)
if getattr(res_pose, "pose_landmarks", None):
pose_landmarks = res_pose.pose_landmarks
pose_features = utils.extract_pose_features(pose_landmarks.landmark)
pose_prediction = pose_model.predict([pose_features])[0]
pose_probs = pose_model.predict_proba([pose_features])[0]
# Draw pose landmarks on the frame
mp.solutions.drawing_utils.draw_landmarks(
img, pose_landmarks, mp_pose.POSE_CONNECTIONS
)
except Exception as e:
print(f"Feature extraction error: {e}")
return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated
# Gathering all class labels used by the three models (normalize as strings)
all_classes = sorted(
set(map(str, left_model.classes_)).union(
set(map(str, right_model.classes_))).union(
set(map(str, pose_model.classes_)))
)
# Helper to align probabilities to all_classes (returns numpy array of len(all_classes))
def align_probs(probs, model_classes):
aligned = np.zeros(len(all_classes), dtype=float)
if probs is None:
return aligned
# Normalize keys to lowercase strings for robust matching
model_cls_list = [str(c).lower() for c in model_classes]
probs_arr = np.array(probs, dtype=float)
if len(model_cls_list) != len(probs_arr):
print(f"Warning: model_classes length ({len(model_cls_list)}) != probs length ({len(probs_arr)})")
mapping = {mc: p for mc, p in zip(model_cls_list, probs_arr)}
aligned = np.array([mapping.get(str(cls).lower(), 0.0) for cls in all_classes], dtype=float) * 100.0
return aligned
# Align probabilities for each source ensuring identical shapes
left_probs_aligned = align_probs(left_probs, getattr(left_model, 'classes_', []))
right_probs_aligned = align_probs(right_probs, getattr(right_model, 'classes_', []))
pose_probs_aligned = align_probs(pose_probs, getattr(pose_model, 'classes_', []))
# Determine the number of available sources (hand(s)/pose)
num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
if num_sources == 0:
return {"message": "No valid data", "frame_count": frame_count}, annotated
# Now safe to compute averaged probabilities (aligned arrays have same shape)
avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100.0 * num_sources)
# Convert these averages into final percentages based on custom thresholds
avg_probs = calulating_percentage(avg, all_classes)
if accumulated_probs is None:
accumulated_probs = np.zeros_like(avg_probs)
accumulated_probs += avg_probs
frame_count += 1
# Updating the final prediction text after a fixed no. of frames
if frame_count == 5:
max_idx = np.argmax(accumulated_probs)
pred = all_classes[max_idx]
accumulated_probs = None
frame_count = 0
return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated
# Rest detection parameters
REST_SPEED_THRESHOLD :float = 30.0 # pixels/second; lower => more sensitive to rest
MIN_POINTS_FOR_REST :int = 2
REQUIRED_CONSECUTIVE_FRAMES :int = 3
REST_DELAY = 2
USE_REST: bool = False
app = Flask(__name__)
# Queue for video frames to be displayed
frame_queue = Queue(maxsize=10)
# Global variables for frame accumulation
accumulated_probs = None
frame_count:int = 0
model_type = "Custom"
# Load models once at startup
left_model, right_model, pose_model = load_models()
# Initialize MediaPipe solutions
mp_hands = mp.solutions.hands
mp_pose = mp.solutions.pose
hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5)
pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7)
utils = Utils(REST_SPEED_THRESHOLD, MIN_POINTS_FOR_REST, REQUIRED_CONSECUTIVE_FRAMES)
# trainer = Trainer(hands,pose)
@app.route('/')
def home():
return f"<h1>Signify API</h1>"
# @app.route('/customTrain', methods= ["POST"])
# def custom_model():
# """
# Trigger custom training (POST)
# Always returns a valid Flask response to avoid None being returned.
# """
# try:
# status = trainer.run_training()
# if status:
# print("Model training completed")
# return jsonify({"status": "success", "message": "Custom model training completed and switched"}), 200
# else:
# return jsonify({"status": "failed", "message": "Custom training did not complete successfully"}), 500
# except Exception as e:
# return jsonify({"status": "error", "message": f"Error training model: {str(e)}"}), 500
@app.route('/switchModel', methods=['POST', 'GET'])
def switch_model():
"""Switch the model based on the response received from the model."""
try:
data = request.get_json(silent=True) or {}
requested = data.get('modelType', 'Default') if isinstance(data, dict) else 'Default'
req_norm = (requested or "Default").strip().lower()
# Normalize into exact 'Default' or 'Custom' strings used by load_models
if req_norm == 'custom':
chosen = 'Custom'
else:
chosen = 'Default'
switch_model_internal(chosen)
return jsonify({"status": "success", "message": f"Switched to model type {chosen}"}), 200
except Exception as e:
return jsonify({"status": "error", "message": f"Error switching model: {str(e)}"}), 500
def switch_model_internal(model_type: str):
"""Internal function to switch models"""
global left_model, right_model, pose_model
left_model, right_model, pose_model = load_models(model_type)
print(f"Models switched to: {model_type}")
@app.route('/processFrame', methods=['POST'])
def process_frame():
"""Receive a video frame and return a prediction."""
try:
if 'frame' not in request.files:
return jsonify({"status": "error", "message": "No frame file received"}), 400
file = request.files['frame']
img_data = file.read()
np_arr = np.frombuffer(img_data, np.uint8)
img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
if img is None:
return jsonify({"status": "error", "message": "Failed to decode image"}), 400
# Rotate the frame 90 degrees clockwise
img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
if frame_queue.full():
try:
# Non-blocking get to remove the oldest frame
frame_queue.get_nowait()
except Empty:
pass
# Process the image to get a prediction
result, annotated_img = process_image(img)
if result.get("USE_REST"):
result, annotated_img = process_image(img, USE_REST=result["USE_REST"])
if result.get("prediction"):
# Clear the queue as we have a final prediction
while not frame_queue.empty():
try:
frame_queue.get_nowait()
except Empty:
break
text = f"Prediction: {result['prediction']}"
cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
frame_queue.put(annotated_img)
return jsonify({
"status": "success",
"prediction": result["prediction"]
}, 200)
else:
text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)"
cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
frame_queue.put(annotated_img)
return jsonify({
"status": "collecting",
"message": result.get("message", "Processing frames"),
"frame_count": result.get("frame_count", 0)
}, 200)
except Exception as e:
return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500
@app.route('/reset', methods=['POST'])
def reset_accumulation():
"""Reset the frame accumulation"""
global accumulated_probs, frame_count
accumulated_probs = None
frame_count = 0
return jsonify({"status": "success", "message": "Accumulation reset"}), 200
@app.route('/health', methods=['GET'])
def health_check():
"""Health check endpoint"""
return jsonify({
"status": "healthy",
"frame_count": frame_count,
"models_loaded": True
}), 200
print("Starting prediction API server...")
print("Endpoints available:")
print("- POST /processFrame - Process received frame and return predictions")
print("- POST /customTrain - Train a custom model")
print("- POST /switchModel - Switch between models")
print("- POST /reset - Reset frame accumulation")
print("- GET /health - Health check")
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)
if __name__ != '__main__':
from asgiref.wsgi import WsgiToAsgi
app = WsgiToAsgi(app)
|