#API from flask import Flask, request, jsonify # ML import pickle import numpy as np from matplotlib import pyplot as plt import mediapipe as mp import cv2 # System modules import threading from queue import Queue, Empty import warnings import os import warnings warnings.filterwarnings("ignore") #Custom modules from utils import Utils from CustomTrainer import Trainer # TO DO # Switching the model based on the response received from the model def calulating_percentage(avg, all_classes): """ Given a list of average probabilities (avg) and corresponding class labels (all_classes), returns a list of percentage values adjusted by class-specific thresholds. """ individual_threshold = { "sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9, "paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9, "accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9, "poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9, "important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9, "pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9, "school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9, "slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9, "poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9 } threshold_percentage = [] for score, cls in zip(avg, all_classes): threshold_val = individual_threshold.get(cls.lower(), 1.0) threshold_percentage.append(score * 100 / threshold_val) return threshold_percentage def load_models(model_type: str = "Default"): """ Load models for given model_type. Accepts values case-insensitively. If model_type == 'custom' (case-insensitive) -> load custom models, otherwise load default models. Raises FileNotFoundError with helpful message if any model file is missing. """ model_type_normalized = (model_type or "Default").strip().lower() if model_type_normalized == "custom": left_model_filename = r'./Custom_Dataset/Models/left_model_new.p' right_model_filename = r'./Custom_Dataset/Models/right_model_new.p' pose_model_filename = r'./Custom_Dataset/Models/pose_model_new.p' else: left_model_filename = r'./Models/left_model.p' right_model_filename = r'./Models/right_model.p' pose_model_filename = r'./Models/pose_model.p' def load_model(filename): if not os.path.exists(filename): raise FileNotFoundError(f"Model file not found: {filename}") with open(filename, 'rb') as f: model_data = pickle.load(f) # Support both {'model': model} and plain model object pickles if isinstance(model_data, dict) and 'model' in model_data: return model_data['model'] return model_data try: return ( load_model(left_model_filename), load_model(right_model_filename), load_model(pose_model_filename) ) except Exception as e: print(f"Failed to load models for type '{model_type}': {e}") raise def process_image(img, USE_REST:bool = False): """Process a single image (np.array) to get a prediction.""" global accumulated_probs, frame_count pred = None left_probs, right_probs, pose_probs = None, None, None img.flags.writeable = False img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) res_pose = pose.process(img_rgb) res_hands = hands.process(img_rgb) img.flags.writeable = True annotated = img.copy() try: if USE_REST: resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None) else: resting = False except Exception as e: print(f"Rest detection error: {e}") resting = False if resting: return {"prediction": "rest"}, annotated try: if getattr(res_hands, "multi_hand_landmarks", None): for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness): label = handedness.classification[0].label features = utils.extract_hand_features( hand_landmarks.landmark, res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else [] ) if label == 'Left': left_probs = left_model.predict_proba([features])[0] elif label == 'Right': right_probs = right_model.predict_proba([features])[0] # Draw hand landmarks on the frame mp.solutions.drawing_utils.draw_landmarks( img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS ) if getattr(res_pose, "pose_landmarks", None): pose_landmarks = res_pose.pose_landmarks pose_features = utils.extract_pose_features(pose_landmarks.landmark) pose_prediction = pose_model.predict([pose_features])[0] pose_probs = pose_model.predict_proba([pose_features])[0] # Draw pose landmarks on the frame mp.solutions.drawing_utils.draw_landmarks( img, pose_landmarks, mp_pose.POSE_CONNECTIONS ) except Exception as e: print(f"Feature extraction error: {e}") return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated # Gathering all class labels used by the three models (normalize as strings) all_classes = sorted( set(map(str, left_model.classes_)).union( set(map(str, right_model.classes_))).union( set(map(str, pose_model.classes_))) ) # Helper to align probabilities to all_classes (returns numpy array of len(all_classes)) def align_probs(probs, model_classes): aligned = np.zeros(len(all_classes), dtype=float) if probs is None: return aligned # Normalize keys to lowercase strings for robust matching model_cls_list = [str(c).lower() for c in model_classes] probs_arr = np.array(probs, dtype=float) if len(model_cls_list) != len(probs_arr): print(f"Warning: model_classes length ({len(model_cls_list)}) != probs length ({len(probs_arr)})") mapping = {mc: p for mc, p in zip(model_cls_list, probs_arr)} aligned = np.array([mapping.get(str(cls).lower(), 0.0) for cls in all_classes], dtype=float) * 100.0 return aligned # Align probabilities for each source ensuring identical shapes left_probs_aligned = align_probs(left_probs, getattr(left_model, 'classes_', [])) right_probs_aligned = align_probs(right_probs, getattr(right_model, 'classes_', [])) pose_probs_aligned = align_probs(pose_probs, getattr(pose_model, 'classes_', [])) # Determine the number of available sources (hand(s)/pose) num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs]) if num_sources == 0: return {"message": "No valid data", "frame_count": frame_count}, annotated # Now safe to compute averaged probabilities (aligned arrays have same shape) avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100.0 * num_sources) # Convert these averages into final percentages based on custom thresholds avg_probs = calulating_percentage(avg, all_classes) if accumulated_probs is None: accumulated_probs = np.zeros_like(avg_probs) accumulated_probs += avg_probs frame_count += 1 # Updating the final prediction text after a fixed no. of frames if frame_count == 5: max_idx = np.argmax(accumulated_probs) pred = all_classes[max_idx] accumulated_probs = None frame_count = 0 return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated # Rest detection parameters REST_SPEED_THRESHOLD :float = 30.0 # pixels/second; lower => more sensitive to rest MIN_POINTS_FOR_REST :int = 2 REQUIRED_CONSECUTIVE_FRAMES :int = 3 REST_DELAY = 2 USE_REST: bool = False app = Flask(__name__) # Queue for video frames to be displayed frame_queue = Queue(maxsize=10) # Global variables for frame accumulation accumulated_probs = None frame_count:int = 0 model_type = "Custom" # Load models once at startup left_model, right_model, pose_model = load_models() # Initialize MediaPipe solutions mp_hands = mp.solutions.hands mp_pose = mp.solutions.pose hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5) pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) utils = Utils(REST_SPEED_THRESHOLD, MIN_POINTS_FOR_REST, REQUIRED_CONSECUTIVE_FRAMES) # trainer = Trainer(hands,pose) @app.route('/') def home(): return f"

Signify API

" # @app.route('/customTrain', methods= ["POST"]) # def custom_model(): # """ # Trigger custom training (POST) # Always returns a valid Flask response to avoid None being returned. # """ # try: # status = trainer.run_training() # if status: # print("Model training completed") # return jsonify({"status": "success", "message": "Custom model training completed and switched"}), 200 # else: # return jsonify({"status": "failed", "message": "Custom training did not complete successfully"}), 500 # except Exception as e: # return jsonify({"status": "error", "message": f"Error training model: {str(e)}"}), 500 @app.route('/switchModel', methods=['POST', 'GET']) def switch_model(): """Switch the model based on the response received from the model.""" try: data = request.get_json(silent=True) or {} requested = data.get('modelType', 'Default') if isinstance(data, dict) else 'Default' req_norm = (requested or "Default").strip().lower() # Normalize into exact 'Default' or 'Custom' strings used by load_models if req_norm == 'custom': chosen = 'Custom' else: chosen = 'Default' switch_model_internal(chosen) return jsonify({"status": "success", "message": f"Switched to model type {chosen}"}), 200 except Exception as e: return jsonify({"status": "error", "message": f"Error switching model: {str(e)}"}), 500 def switch_model_internal(model_type: str): """Internal function to switch models""" global left_model, right_model, pose_model left_model, right_model, pose_model = load_models(model_type) print(f"Models switched to: {model_type}") @app.route('/processFrame', methods=['POST']) def process_frame(): """Receive a video frame and return a prediction.""" try: if 'frame' not in request.files: return jsonify({"status": "error", "message": "No frame file received"}), 400 file = request.files['frame'] img_data = file.read() np_arr = np.frombuffer(img_data, np.uint8) img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) if img is None: return jsonify({"status": "error", "message": "Failed to decode image"}), 400 # Rotate the frame 90 degrees clockwise img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE) if frame_queue.full(): try: # Non-blocking get to remove the oldest frame frame_queue.get_nowait() except Empty: pass # Process the image to get a prediction result, annotated_img = process_image(img) if result.get("USE_REST"): result, annotated_img = process_image(img, USE_REST=result["USE_REST"]) if result.get("prediction"): # Clear the queue as we have a final prediction while not frame_queue.empty(): try: frame_queue.get_nowait() except Empty: break text = f"Prediction: {result['prediction']}" cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA) frame_queue.put(annotated_img) return jsonify({ "status": "success", "prediction": result["prediction"] }, 200) else: text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)" cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA) frame_queue.put(annotated_img) return jsonify({ "status": "collecting", "message": result.get("message", "Processing frames"), "frame_count": result.get("frame_count", 0) }, 200) except Exception as e: return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500 @app.route('/reset', methods=['POST']) def reset_accumulation(): """Reset the frame accumulation""" global accumulated_probs, frame_count accumulated_probs = None frame_count = 0 return jsonify({"status": "success", "message": "Accumulation reset"}), 200 @app.route('/health', methods=['GET']) def health_check(): """Health check endpoint""" return jsonify({ "status": "healthy", "frame_count": frame_count, "models_loaded": True }), 200 print("Starting prediction API server...") print("Endpoints available:") print("- POST /processFrame - Process received frame and return predictions") print("- POST /customTrain - Train a custom model") print("- POST /switchModel - Switch between models") print("- POST /reset - Reset frame accumulation") print("- GET /health - Health check") if __name__ == '__main__': app.run(host='0.0.0.0', port=5000, debug=False, threaded=True) if __name__ != '__main__': from asgiref.wsgi import WsgiToAsgi app = WsgiToAsgi(app)