File size: 13,390 Bytes
cb8a780
cd1e957
cb8a780
 
cd1e957
 
 
 
 
 
cb8a780
 
 
 
 
 
 
cd1e957
cb8a780
 
 
cd1e957
cb8a780
 
cd1e957
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cb8a780
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cd1e957
 
cb8a780
 
cd1e957
 
cb8a780
 
cd1e957
cb8a780
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
416f492
cb8a780
 
 
 
 
 
 
 
cd1e957
cb8a780
 
 
 
cd1e957
 
 
 
 
 
 
 
 
 
cb8a780
416f492
cd1e957
cb8a780
 
 
 
fef9a8e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
cb8a780
 
 
 
cd1e957
cb8a780
 
 
 
 
 
 
 
 
 
 
cd1e957
cb8a780
cd1e957
cb8a780
 
 
 
 
cd1e957
cb8a780
 
 
 
cd1e957
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36ca911
cd1e957
 
 
cb8a780
 
 
 
 
cd1e957
 
 
 
 
36ca911
cd1e957
 
 
 
 
 
 
 
 
cb8a780
cd1e957
 
 
 
 
 
 
 
 
cb8a780
cd1e957
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8b235ce
 
cb8a780
 
 
8b235ce
 
 
cd1e957
8b235ce
 
 
 
 
f8c9b2e
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
#API
from flask import Flask, request, jsonify

# ML
import pickle
import numpy as np
from matplotlib import pyplot as plt
import mediapipe as mp
import cv2

# System modules
import threading
from queue import Queue, Empty
import warnings
import os
import warnings
warnings.filterwarnings("ignore")

#Custom modules
from utils import Utils
from CustomTrainer import Trainer

# TO DO
# Switching the model based on the response received from the model


def calulating_percentage(avg, all_classes):
    """
    Given a list of average probabilities (avg) and corresponding class labels (all_classes),
    returns a list of percentage values adjusted by class-specific thresholds.
    """
    
    individual_threshold = {
        "sun": 0.9, "help": 0.9, "teacher": 0.9, "support": 0.9,
        "paper": 0.9, "love": 0.9, "dance": 0.9, "water": 0.9,
        "accident": 0.9, "yes": 0.9, "thick": 0.9, "high": 0.9,
        "poor": 0.9, "i": 0.9, "my": 0.9, "important_1": 0.9,
        "important_2": 0.9, "deaf": 0.9, "winner": 0.9, "eat": 0.9,
        "pizza": 0.9, "go": 0.9, "isl": 0.9, "friend": 0.9,
        "school": 0.9, "deep": 0.9, "loud": 0.9, "flat": 0.9,
        "slow": 0.9, "sad": 0.9, "soft": 0.9, "happy": 0.9,
        "poot": 0.9, "quiet": 0.9, "book": 0.9, "woman": 0.9
    }

    threshold_percentage = []
    for score, cls in zip(avg, all_classes):
        threshold_val = individual_threshold.get(cls.lower(), 1.0)
        threshold_percentage.append(score * 100 / threshold_val)

    return threshold_percentage

def load_models(model_type: str = "Default"):
    """
    Load models for given model_type. Accepts values case-insensitively.
    If model_type == 'custom' (case-insensitive) -> load custom models,
    otherwise load default models. Raises FileNotFoundError with helpful message
    if any model file is missing.
    """
    model_type_normalized = (model_type or "Default").strip().lower()
    if model_type_normalized == "custom":
        left_model_filename = r'./Custom_Dataset/Models/left_model_new.p'
        right_model_filename = r'./Custom_Dataset/Models/right_model_new.p'
        pose_model_filename = r'./Custom_Dataset/Models/pose_model_new.p'
    else:
        left_model_filename = r'./Models/left_model.p'
        right_model_filename = r'./Models/right_model.p'
        pose_model_filename = r'./Models/pose_model.p'

    def load_model(filename):
        if not os.path.exists(filename):
            raise FileNotFoundError(f"Model file not found: {filename}")
        with open(filename, 'rb') as f:
            model_data = pickle.load(f)
        # Support both {'model': model} and plain model object pickles
        if isinstance(model_data, dict) and 'model' in model_data:
            return model_data['model']
        return model_data

    try:
        return (
            load_model(left_model_filename),
            load_model(right_model_filename),
            load_model(pose_model_filename)
        )
    except Exception as e:
        print(f"Failed to load models for type '{model_type}': {e}")
        raise

def process_image(img, USE_REST:bool = False):
	"""Process a single image (np.array) to get a prediction."""
	global accumulated_probs, frame_count

	pred = None
	left_probs, right_probs, pose_probs = None, None, None

	img.flags.writeable = False
	img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
	res_pose = pose.process(img_rgb)
	res_hands = hands.process(img_rgb)
	img.flags.writeable = True
	annotated = img.copy()
	

	try:
		if USE_REST:
			resting = utils.is_resting(res_hands, annotated.shape, rest_delay_seconds=REST_DELAY, fps=None)
		else:
			resting = False
	except Exception as e:
		print(f"Rest detection error: {e}")
		resting = False

	if resting:
		return {"prediction": "rest"}, annotated

	try:
		if getattr(res_hands, "multi_hand_landmarks", None):
			for hand_landmarks, handedness in zip(res_hands.multi_hand_landmarks, res_hands.multi_handedness):
				label = handedness.classification[0].label
				features = utils.extract_hand_features(
					hand_landmarks.landmark,
					res_pose.pose_landmarks.landmark if res_pose.pose_landmarks else []
				)

				if label == 'Left':
					left_probs = left_model.predict_proba([features])[0]
				elif label == 'Right':
					right_probs = right_model.predict_proba([features])[0]

				# Draw hand landmarks on the frame
				mp.solutions.drawing_utils.draw_landmarks(
					img, hand_landmarks, mp.solutions.hands.HAND_CONNECTIONS
				)

		if getattr(res_pose, "pose_landmarks", None):
			pose_landmarks = res_pose.pose_landmarks
			pose_features = utils.extract_pose_features(pose_landmarks.landmark)
			pose_prediction = pose_model.predict([pose_features])[0]
			pose_probs = pose_model.predict_proba([pose_features])[0]
			# Draw pose landmarks on the frame
			mp.solutions.drawing_utils.draw_landmarks(
				img, pose_landmarks, mp_pose.POSE_CONNECTIONS
			)

	except Exception as e:
		print(f"Feature extraction error: {e}")
		return {"message": "Feature extraction failed", "frame_count": frame_count}, annotated

	# Gathering all class labels used by the three models (normalize as strings)
	all_classes = sorted(
		set(map(str, left_model.classes_)).union(
		set(map(str, right_model.classes_))).union(
		set(map(str, pose_model.classes_)))
	)

	# Helper to align probabilities to all_classes (returns numpy array of len(all_classes))
	def align_probs(probs, model_classes):
		aligned = np.zeros(len(all_classes), dtype=float)
		if probs is None:
			return aligned
		# Normalize keys to lowercase strings for robust matching
		model_cls_list = [str(c).lower() for c in model_classes]
		probs_arr = np.array(probs, dtype=float)
		if len(model_cls_list) != len(probs_arr):
			print(f"Warning: model_classes length ({len(model_cls_list)}) != probs length ({len(probs_arr)})")
		mapping = {mc: p for mc, p in zip(model_cls_list, probs_arr)}
		aligned = np.array([mapping.get(str(cls).lower(), 0.0) for cls in all_classes], dtype=float) * 100.0
		return aligned

	# Align probabilities for each source ensuring identical shapes
	left_probs_aligned = align_probs(left_probs, getattr(left_model, 'classes_', []))
	right_probs_aligned = align_probs(right_probs, getattr(right_model, 'classes_', []))
	pose_probs_aligned = align_probs(pose_probs, getattr(pose_model, 'classes_', []))

	# Determine the number of available sources (hand(s)/pose)
	num_sources = sum(prob is not None for prob in [left_probs, right_probs, pose_probs])
	if num_sources == 0:
		return {"message": "No valid data", "frame_count": frame_count}, annotated

	# Now safe to compute averaged probabilities (aligned arrays have same shape)
	avg = (left_probs_aligned + right_probs_aligned + pose_probs_aligned) / (100.0 * num_sources)

	# Convert these averages into final percentages based on custom thresholds
	avg_probs = calulating_percentage(avg, all_classes)

	if accumulated_probs is None:
		accumulated_probs = np.zeros_like(avg_probs)
	accumulated_probs += avg_probs
	frame_count += 1

	# Updating the final prediction text after a fixed no. of frames
	if frame_count == 5:
		max_idx = np.argmax(accumulated_probs)
		pred = all_classes[max_idx]
		accumulated_probs = None
		frame_count = 0

	return ({"prediction": pred} if pred else {"message": "Collecting frames", "frame_count": frame_count}), annotated

# Rest detection parameters
REST_SPEED_THRESHOLD :float = 30.0   # pixels/second; lower => more sensitive to rest
MIN_POINTS_FOR_REST  :int = 2       
REQUIRED_CONSECUTIVE_FRAMES :int = 3  
REST_DELAY = 2
USE_REST: bool = False

app = Flask(__name__)

# Queue for video frames to be displayed
frame_queue = Queue(maxsize=10)

# Global variables for frame accumulation
accumulated_probs = None
frame_count:int = 0
model_type = "Custom"   

# Load models once at startup
left_model, right_model, pose_model = load_models()

# Initialize MediaPipe solutions
mp_hands = mp.solutions.hands
mp_pose = mp.solutions.pose
hands = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.5, min_tracking_confidence=0.5)
pose = mp_pose.Pose(min_detection_confidence=0.7, min_tracking_confidence=0.7)

utils = Utils(REST_SPEED_THRESHOLD, MIN_POINTS_FOR_REST, REQUIRED_CONSECUTIVE_FRAMES)
# trainer = Trainer(hands,pose)

@app.route('/')
def home():
    return f"<h1>Signify API</h1>"

# @app.route('/customTrain', methods= ["POST"])
# def custom_model():
# 	"""
# 	Trigger custom training (POST)   
# 	Always returns a valid Flask response to avoid None being returned.
# 	"""
# 	try:
# 		status = trainer.run_training()
# 		if status:
# 			print("Model training completed")
# 			return jsonify({"status": "success", "message": "Custom model training completed and switched"}), 200
# 		else:
# 			return jsonify({"status": "failed", "message": "Custom training did not complete successfully"}), 500
# 	except Exception as e:
# 		return jsonify({"status": "error", "message": f"Error training model: {str(e)}"}), 500

@app.route('/switchModel', methods=['POST', 'GET'])
def switch_model():
    """Switch the model based on the response received from the model."""
    try:
        data = request.get_json(silent=True) or {}
        requested = data.get('modelType', 'Default') if isinstance(data, dict) else 'Default'
        req_norm = (requested or "Default").strip().lower()
        # Normalize into exact 'Default' or 'Custom' strings used by load_models
        if req_norm == 'custom':
            chosen = 'Custom'
        else:
            chosen = 'Default'

        switch_model_internal(chosen)
        return jsonify({"status": "success", "message": f"Switched to model type {chosen}"}), 200
    except Exception as e:
        return jsonify({"status": "error", "message": f"Error switching model: {str(e)}"}), 500

def switch_model_internal(model_type: str):
    """Internal function to switch models"""
    global left_model, right_model, pose_model
    left_model, right_model, pose_model = load_models(model_type)
    print(f"Models switched to: {model_type}")


@app.route('/processFrame', methods=['POST'])
def process_frame():
    """Receive a video frame and return a prediction."""
    try:
        if 'frame' not in request.files:
            return jsonify({"status": "error", "message": "No frame file received"}), 400
        
        file = request.files['frame']
        img_data = file.read()
        
        np_arr = np.frombuffer(img_data, np.uint8)
        img = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
        
        if img is None:
            return jsonify({"status": "error", "message": "Failed to decode image"}), 400
        
        # Rotate the frame 90 degrees clockwise
        img = cv2.rotate(img, cv2.ROTATE_90_CLOCKWISE)
        
        if frame_queue.full():
            try:
                # Non-blocking get to remove the oldest frame
                frame_queue.get_nowait()
            except Empty:
                pass  
            
        # Process the image to get a prediction
        result, annotated_img = process_image(img)

        if result.get("USE_REST"):
            result, annotated_img = process_image(img, USE_REST=result["USE_REST"])

        if result.get("prediction"):
            # Clear the queue as we have a final prediction
            while not frame_queue.empty():
                try:
                    frame_queue.get_nowait()
                except Empty:
                    break
            
            text = f"Prediction: {result['prediction']}"
            cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
            frame_queue.put(annotated_img)

            return jsonify({
                "status": "success",
                "prediction": result["prediction"]
            }, 200)
        else:
            text = f"{result.get('message', 'Processing')} ({result.get('frame_count', 0)}/5)"
            cv2.putText(annotated_img, text, (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
            frame_queue.put(annotated_img)

            return jsonify({
                "status": "collecting",
                "message": result.get("message", "Processing frames"),
                "frame_count": result.get("frame_count", 0)
            }, 200)

    except Exception as e:
        return jsonify({"status": "error", "message": f"Processing error: {str(e)}"}), 500

@app.route('/reset', methods=['POST'])
def reset_accumulation():
    """Reset the frame accumulation"""
    global accumulated_probs, frame_count
    accumulated_probs = None
    frame_count = 0
    return jsonify({"status": "success", "message": "Accumulation reset"}), 200

@app.route('/health', methods=['GET'])
def health_check():
    """Health check endpoint"""
    return jsonify({
        "status": "healthy", 
        "frame_count": frame_count,
        "models_loaded": True
    }), 200

print("Starting prediction API server...")
print("Endpoints available:")
print("- POST /processFrame - Process received frame and return predictions")
print("- POST /customTrain - Train a custom model")
print("- POST /switchModel - Switch between models")
print("- POST /reset - Reset frame accumulation")
print("- GET /health - Health check")

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)
    
if __name__ != '__main__':
    from asgiref.wsgi import WsgiToAsgi
    app = WsgiToAsgi(app)