Spaces:
Runtime error
Runtime error
Download app.py from Philosia-codecult/Signify: direct link, hf CLI and curl.
- Browser
- Download file 13.4 kB
-
https://huggingface.co/spaces/Philosia-codecult/Signify/resolve/main/app.py
- Command line
-
hf download hf://spaces/Philosia-codecult/Signify/app.py
-
curl -L -o app.py https://huggingface.co/spaces/Philosia-codecult/Signify/resolve/main/app.py
13.4 kB
| #API | |
| from flask import Flask, request, jsonify | |
| # ML | |
| import pickle | |
| import numpy as np | |
| from matplotlib import pyplot as plt | |
| import mediapipe as mp | |
| import cv2 | |
| # System modules | |
| import threading | |
| from queue import Queue, Empty | |
| import warnings | |
| import os | |
| import warnings | |
| warnings.filterwarnings("ignore") | |
| #Custom modules | |
| from utils import Utils | |
| from CustomTrainer import Trainer | |
| # TO DO | |
| # Switching the model based on the response received from the model | |
| def calulating_percentage(avg, all_classes): | |
| """ | |
| Given a list of average probabilities (avg) and corresponding class labels (all_classes), | |
| returns a list of percentage values adjusted by class-specific thresholds. | |
| """ | |
| individual_threshold = { | |
| "sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9, | |
| "paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9, | |
| "accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9, | |
| "poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9, | |
| "important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9, | |
| "pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9, | |
| "school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9, | |
| "slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9, | |
| "poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9 | |
| } | |
| threshold_percentage = [] | |
| for score, cls in zip(avg, all_classes): | |
| threshold_val = individual_threshold.get(cls.lower(), 1.0) | |
| threshold_percentage.append(score * 100 / threshold_val) | |
| return threshold_percentage | |
| def load_models(model_type: str = "Default"): | |
| """ | |
| Load models for given model_type. Accepts values case-insensitively. | |
| If model_type == 'custom' (case-insensitive) -> load custom models, | |
| otherwise load default models. Raises FileNotFoundError with helpful message | |
| if any model file is missing. | |
| """ | |
| model_type_normalized = (model_type or "Default").strip().lower() | |
| if model_type_normalized == "custom": | |
| left_model_filename = r'./Custom_Dataset/Models/left_model_new.p' | |
| right_model_filename = r'./Custom_Dataset/Models/right_model_new.p' | |
| pose_model_filename = r'./Custom_Dataset/Models/pose_model_new.p' | |
| else: | |
| left_model_filename = r'./Models/left_model.p' | |
| right_model_filename = r'./Models/right_model.p' | |
| pose_model_filename = r'./Models/pose_model.p' | |
| def load_model(filename): | |
| if not os.path.exists(filename): | |
| raise FileNotFoundError(f"Model file not found: {filename}") | |
| with open(filename, 'rb') as f: | |
| model_data = pickle.load(f) | |
| # Support both {'model': model} and plain model object pickles | |
| if isinstance(model_data, dict) and 'model' in model_data: | |
| return model_data['model'] | |
| return model_data | |
| try: | |
| return ( | |
| load_model(left_model_filename), | |
| load_model(right_model_filename), | |
| load_model(pose_model_filename) | |
| ) | |
| except Exception as e: | |
| print(f"Failed to load models for type '{model_type}': {e}") | |
| raise | |
| def process_image(img, USE_REST:bool = False): | |
| """Process a single image (np.array) to get a prediction.""" | |
| global accumulated_probs, frame_count | |
| pred = None | |
| left_probs, right_probs, pose_probs = None, None, None | |
| img.flags.writeable = False | |
| img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) | |
| res_pose = pose.process(img_rgb) | |
| res_hands = hands.process(img_rgb) | |
| img.flags.writeable = True | |
| annotated = img.copy() | |
| try: | |
| if USE_REST: | |
| resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None) | |
| else: | |
| resting = False | |
| except Exception as e: | |
| print(f"Rest detection error: {e}") | |
| resting = False | |
| if resting: | |
| return {"prediction": "rest"}, annotated | |
| try: | |
| if getattr(res_hands, "multi_hand_landmarks", None): | |
| for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness): | |
| label = handedness.classification[0].label | |
| features = utils.extract_hand_features( | |
| hand_landmarks.landmark, | |
| res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else [] | |
| ) | |
| if label == 'Left': | |
| left_probs = left_model.predict_proba([features])[0] | |
| elif label == 'Right': | |
| right_probs = right_model.predict_proba([features])[0] | |
| # Draw hand landmarks on the frame | |
| mp.solutions.drawing_utils.draw_landmarks( | |
| img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS | |
| ) | |
| if getattr(res_pose, "pose_landmarks", None): | |
| pose_landmarks = res_pose.pose_landmarks | |
| pose_features = utils.extract_pose_features(pose_landmarks.landmark) | |
| pose_prediction = pose_model.predict([pose_features])[0] | |
| pose_probs = pose_model.predict_proba([pose_features])[0] | |
| # Draw pose landmarks on the frame | |
| mp.solutions.drawing_utils.draw_landmarks( | |
| img, pose_landmarks, mp_pose.POSE_CONNECTIONS | |
| ) | |
| except Exception as e: | |
| print(f"Feature extraction error: {e}") | |
| return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated | |
| # Gathering all class labels used by the three models (normalize as strings) | |
| all_classes = sorted( | |
| set(map(str, left_model.classes_)).union( | |
| set(map(str, right_model.classes_))).union( | |
| set(map(str, pose_model.classes_))) | |
| ) | |
| # Helper to align probabilities to all_classes (returns numpy array of len(all_classes)) | |
| def align_probs(probs, model_classes): | |
| aligned = np.zeros(len(all_classes), dtype=float) | |
| if probs is None: | |
| return aligned | |
| # Normalize keys to lowercase strings for robust matching | |
| model_cls_list = [str(c).lower() for c in model_classes] | |
| probs_arr = np.array(probs, dtype=float) | |
| if len(model_cls_list) != len(probs_arr): | |
| print(f"Warning: model_classes length ({len(model_cls_list)}) != probs length ({len(probs_arr)})") | |
| mapping = {mc: p for mc, p in zip(model_cls_list, probs_arr)} | |
| aligned = np.array([mapping.get(str(cls).lower(), 0.0) for cls in all_classes], dtype=float) * 100.0 | |
| return aligned | |
| # Align probabilities for each source ensuring identical shapes | |
| left_probs_aligned = align_probs(left_probs, getattr(left_model, 'classes_', [])) | |
| right_probs_aligned = align_probs(right_probs, getattr(right_model, 'classes_', [])) | |
| pose_probs_aligned = align_probs(pose_probs, getattr(pose_model, 'classes_', [])) | |
| # Determine the number of available sources (hand(s)/pose) | |
| num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs]) | |
| if num_sources == 0: | |
| return {"message": "No valid data", "frame_count": frame_count}, annotated | |
| # Now safe to compute averaged probabilities (aligned arrays have same shape) | |
| avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100.0 * num_sources) | |
| # Convert these averages into final percentages based on custom thresholds | |
| avg_probs = calulating_percentage(avg, all_classes) | |
| if accumulated_probs is None: | |
| accumulated_probs = np.zeros_like(avg_probs) | |
| accumulated_probs += avg_probs | |
| frame_count += 1 | |
| # Updating the final prediction text after a fixed no. of frames | |
| if frame_count == 5: | |
| max_idx = np.argmax(accumulated_probs) | |
| pred = all_classes[max_idx] | |
| accumulated_probs = None | |
| frame_count = 0 | |
| return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated | |
| # Rest detection parameters | |
| REST_SPEED_THRESHOLD :float = 30.0 # pixels/second; lower => more sensitive to rest | |
| MIN_POINTS_FOR_REST :int = 2 | |
| REQUIRED_CONSECUTIVE_FRAMES :int = 3 | |
| REST_DELAY = 2 | |
| USE_REST: bool = False | |
| app = Flask(__name__) | |
| # Queue for video frames to be displayed | |
| frame_queue = Queue(maxsize=10) | |
| # Global variables for frame accumulation | |
| accumulated_probs = None | |
| frame_count:int = 0 | |
| model_type = "Custom" | |
| # Load models once at startup | |
| left_model, right_model, pose_model = load_models() | |
| # Initialize MediaPipe solutions | |
| mp_hands = mp.solutions.hands | |
| mp_pose = mp.solutions.pose | |
| hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5) | |
| pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7) | |
| utils = Utils(REST_SPEED_THRESHOLD, MIN_POINTS_FOR_REST, REQUIRED_CONSECUTIVE_FRAMES) | |
| # trainer = Trainer(hands,pose) | |
| def home(): | |
| return f"<h1>Signify API</h1>" | |
| # @app.route('/customTrain', methods= ["POST"]) | |
| # def custom_model(): | |
| # """ | |
| # Trigger custom training (POST) | |
| # Always returns a valid Flask response to avoid None being returned. | |
| # """ | |
| # try: | |
| # status = trainer.run_training() | |
| # if status: | |
| # print("Model training completed") | |
| # return jsonify({"status": "success", "message": "Custom model training completed and switched"}), 200 | |
| # else: | |
| # return jsonify({"status": "failed", "message": "Custom training did not complete successfully"}), 500 | |
| # except Exception as e: | |
| # return jsonify({"status": "error", "message": f"Error training model: {str(e)}"}), 500 | |
| def switch_model(): | |
| """Switch the model based on the response received from the model.""" | |
| try: | |
| data = request.get_json(silent=True) or {} | |
| requested = data.get('modelType', 'Default') if isinstance(data, dict) else 'Default' | |
| req_norm = (requested or "Default").strip().lower() | |
| # Normalize into exact 'Default' or 'Custom' strings used by load_models | |
| if req_norm == 'custom': | |
| chosen = 'Custom' | |
| else: | |
| chosen = 'Default' | |
| switch_model_internal(chosen) | |
| return jsonify({"status": "success", "message": f"Switched to model type {chosen}"}), 200 | |
| except Exception as e: | |
| return jsonify({"status": "error", "message": f"Error switching model: {str(e)}"}), 500 | |
| def switch_model_internal(model_type: str): | |
| """Internal function to switch models""" | |
| global left_model, right_model, pose_model | |
| left_model, right_model, pose_model = load_models(model_type) | |
| print(f"Models switched to: {model_type}") | |
| def process_frame(): | |
| """Receive a video frame and return a prediction.""" | |
| try: | |
| if 'frame' not in request.files: | |
| return jsonify({"status": "error", "message": "No frame file received"}), 400 | |
| file = request.files['frame'] | |
| img_data = file.read() | |
| np_arr = np.frombuffer(img_data, np.uint8) | |
| img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) | |
| if img is None: | |
| return jsonify({"status": "error", "message": "Failed to decode image"}), 400 | |
| # Rotate the frame 90 degrees clockwise | |
| img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE) | |
| if frame_queue.full(): | |
| try: | |
| # Non-blocking get to remove the oldest frame | |
| frame_queue.get_nowait() | |
| except Empty: | |
| pass | |
| # Process the image to get a prediction | |
| result, annotated_img = process_image(img) | |
| if result.get("USE_REST"): | |
| result, annotated_img = process_image(img, USE_REST=result["USE_REST"]) | |
| if result.get("prediction"): | |
| # Clear the queue as we have a final prediction | |
| while not frame_queue.empty(): | |
| try: | |
| frame_queue.get_nowait() | |
| except Empty: | |
| break | |
| text = f"Prediction: {result['prediction']}" | |
| cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA) | |
| frame_queue.put(annotated_img) | |
| return jsonify({ | |
| "status": "success", | |
| "prediction": result["prediction"] | |
| }, 200) | |
| else: | |
| text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)" | |
| cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA) | |
| frame_queue.put(annotated_img) | |
| return jsonify({ | |
| "status": "collecting", | |
| "message": result.get("message", "Processing frames"), | |
| "frame_count": result.get("frame_count", 0) | |
| }, 200) | |
| except Exception as e: | |
| return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500 | |
| def reset_accumulation(): | |
| """Reset the frame accumulation""" | |
| global accumulated_probs, frame_count | |
| accumulated_probs = None | |
| frame_count = 0 | |
| return jsonify({"status": "success", "message": "Accumulation reset"}), 200 | |
| def health_check(): | |
| """Health check endpoint""" | |
| return jsonify({ | |
| "status": "healthy", | |
| "frame_count": frame_count, | |
| "models_loaded": True | |
| }), 200 | |
| print("Starting prediction API server...") | |
| print("Endpoints available:") | |
| print("- POST /processFrame - Process received frame and return predictions") | |
| print("- POST /customTrain - Train a custom model") | |
| print("- POST /switchModel - Switch between models") | |
| print("- POST /reset - Reset frame accumulation") | |
| print("- GET /health - Health check") | |
| if __name__ == '__main__': | |
| app.run(host='0.0.0.0', port=5000, debug=False, threaded=True) | |
| if __name__ != '__main__': | |
| from asgiref.wsgi import WsgiToAsgi | |
| app = WsgiToAsgi(app) | |