perf: fix lag — smaller frames 160x120@35%, flow control, nginx 512k body buffer, 320x240 output

isl_detection.py

@@ -1,24 +1,24 @@
-# isl_detection.py — CLOUD PRODUCTION VERSION (HF SPACES COMPATIBLE)
+# isl_detection.py — CLOUD PRODUCTION VERSION
 #
 # ════════════════════════════════════════════════════════════════════════════
-#  KEY FIXES IN THIS VERSION:
+#  CLOUD FIXES APPLIED (3 changes only — everything else identical):
 #
-#  FIX A — SharedState uses queue.Queue (no Manager, no IPC crash)
-#  FIX B — _cloud_frame is now a dict {"data": ..., "ts": ...} with a lock
-#           so the detection loop and socket handler never race
-#  FIX C — Frame decode is hardened: strips header prefix safely every time
-#  FIX D — Added detailed per-stage debug prints so you can see exactly
-#           where the pipeline breaks from HF Logs tab
-#  FIX E — mediapipe hand + face results checked for None before use
-#  FIX F — model input shape validated before predict (guards wrong .h5)
+#  FIX 1 — pygame TTS: init only in LOCAL mode (crashes on headless server)
+#  FIX 2 — cv2.CAP_DSHOW: Windows-only flag removed for Linux cloud server
+#  FIX 3 — SharedMemory cleanup on startup (prevents crash on restart)
 #
-#  All original features preserved:
+#  LOCAL CAMERA FIXES (built-in laptop webcam static/noise):
+#  FIX 4 — MJPG codec + 30-frame flush to clear DirectShow buffer on Windows
+#  FIX 5 — CAP_PROP_BUFFERSIZE set before flush (not after)
+#
+#  ALL original features preserved:
 #  ✅ Emotion detection v3 (happy/sad/angry/surprise/neutral)
 #  ✅ Letter hold timer, same-letter cooldown, confidence buffer
-#  ✅ CLOUD mode browser frame ingestion
-#  ✅ gTTS speak button (disabled in cloud)
+#  ✅ SharedMemory zero-copy frame transfer
+#  ✅ CLOUD mode browser frame ingestion (_cloud_frame)
+#  ✅ gTTS speak button (disabled silently in cloud — browser TTS handles it)
 #  ✅ Word suggestions, auto-TTS, reset, backspace, accept_suggestion
-#  ✅ TARGET_FPS=15, EMOTION_PROCESS_INTERVAL=5
+#  ✅ Oracle Free Tier safe: TARGET_FPS=15, EMOTION_PROCESS_INTERVAL=5
 # ════════════════════════════════════════════════════════════════════════════
 
 import cv2
@@ -33,27 +33,15 @@ import base64
 import platform
 import threading
 from collections import deque, defaultdict, Counter
-from multiprocessing import Value
+from multiprocessing import Manager, Value, shared_memory
 import ctypes
-import queue as queue_module
 
-os.environ["CUDA_VISIBLE_DEVICES"]  = "-1"
-os.environ["TF_CPP_MIN_LOG_LEVEL"]  = "3"
-os.environ["MEDIAPIPE_DISABLE_GPU"] = "1"
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 
 from tensorflow.keras.models import load_model
-
-try:
-    import pygame
-    _PYGAME_AVAILABLE = True
-except ImportError:
-    _PYGAME_AVAILABLE = False
-
-try:
-    from gtts import gTTS
-    _GTTS_AVAILABLE = True
-except ImportError:
-    _GTTS_AVAILABLE = False
+from gtts import gTTS
+import pygame
 
 try:
     import translation
@@ -64,20 +52,22 @@ except ImportError:
 # ─────────────────────────────────────────────────────────────
 #  CONFIG
 # ─────────────────────────────────────────────────────────────
-RUN_MODE          = os.getenv("RUN_MODE", "CLOUD")
+RUN_MODE          = os.getenv("RUN_MODE", "LOCAL")
 ISL_MODEL_PATH    = "model.h5"
 CAM_INDEX         = 0
 
+# ── Letter timing ──────────────────────────────────────────────────────────
 LETTER_HOLD_SEC          = 0.9
 SAME_LETTER_COOLDOWN_SEC = 1.2
 
+# ── Emotion thresholds ────────────────────────────────────────────────────
 SMOOTHING_FRAMES         = 2
 MIN_CONF_TO_SHOW         = 0.06
 EMOTIONS                 = ["happy", "sad", "angry", "surprise", "neutral"]
 
 CONF_THRESHOLD           = 0.32
 SPACE_THRESHOLD          = 18
-ENABLE_AUTO_TTS          = False   # disabled in cloud; browser uses SpeechSynthesis
+ENABLE_AUTO_TTS          = True
 TTS_LANGUAGE             = "en"
 
 SHOW_LETTER_OVERLAY      = True
@@ -98,46 +88,28 @@ CAMERA_RESOLUTION        = (640, 480)
 EMOTION_PROCESS_INTERVAL = 5
 FRAME_W, FRAME_H         = 640, 480
 
+
 # ─────────────────────────────────────────────────────────────
 #  SHARED STATE
-#  Uses stdlib queue.Queue — no Manager(), no IPC, no crashes.
-#  _cloud_frame_lock guards the raw frame slot so the eventlet
-#  greenlet (on_browser_frame) and the detection thread never race.
 # ─────────────────────────────────────────────────────────────
 class SharedState:
     def __init__(self):
-        self.ui_queue          = queue_module.Queue(maxsize=200)
-        self.command_queue     = queue_module.Queue(maxsize=20)
-        self.processing_fps    = Value(ctypes.c_double, 0.0)
-        self._cloud_frame      = None          # raw base64 string (header stripped)
-        self._cloud_frame_lock = threading.Lock()
-        self._frame_count_in   = 0             # debug: total frames received
-        self._frame_count_proc = 0             # debug: total frames processed
-
-    def put_frame(self, b64_data: str):
-        """Called by socket handler — strips header and stores atomically."""
-        if not b64_data:
-            return
-        # Strip data URI header if present (e.g. "data:image/jpeg;base64,...")
-        if "," in b64_data:
-            b64_data = b64_data.split(",", 1)[1]
-        with self._cloud_frame_lock:
-            self._cloud_frame = b64_data
-            self._frame_count_in += 1
-
-    def take_frame(self):
-        """Called by detection thread — returns base64 string or None."""
-        with self._cloud_frame_lock:
-            data = self._cloud_frame
-            self._cloud_frame = None
-            return data
+        manager = Manager()
+        self.ui_queue       = manager.Queue(200)
+        self.command_queue  = manager.Queue(20)
+        self.processing_fps = Value(ctypes.c_double, 0.0)
 
 
 # ─────────────────────────────────────────────────────────────
-#  TTS ENGINE (LOCAL only)
+#  TTS ENGINE
+#  FIX 1: pygame.mixer.init() only runs in LOCAL mode.
+#          On cloud (headless Linux) there is no audio device —
+#          init() would crash the entire process at import time.
+#          In CLOUD mode TTS is silently disabled here; the browser
+#          handles speech via the Web Speech API on the client side.
 # ─────────────────────────────────────────────────────────────
 TTS_ENABLED = False
-if RUN_MODE == "LOCAL" and _PYGAME_AVAILABLE and _GTTS_AVAILABLE:
+if RUN_MODE == "LOCAL":
     try:
         pygame.mixer.init()
         TTS_ENABLED = True
@@ -145,7 +117,7 @@ if RUN_MODE == "LOCAL" and _PYGAME_AVAILABLE and _GTTS_AVAILABLE:
     except Exception as e:
         print(f"⚠️  TTS init warning: {e}")
 else:
-    print("ℹ️  TTS disabled (CLOUD/headless mode)")
+    print("ℹ️  TTS disabled (CLOUD/headless mode) — browser handles speech")
 
 speaking_word = ""
 pause_audio   = False
@@ -153,6 +125,8 @@ pause_audio   = False
 
 def speak_text(text, lang=TTS_LANGUAGE):
     global speaking_word
+
+    # FIX 1 continued: guard every call — silently skip on cloud
     if not TTS_ENABLED or not text or not text.strip():
         return
 
@@ -189,22 +163,6 @@ def speak_text(text, lang=TTS_LANGUAGE):
     threading.Thread(target=_speak, daemon=True).start()
 
 
-# ─────────────────────────────────────────────────────────────
-#  FRAME DECODE HELPER
-#  Accepts raw base64 (no header).  Returns BGR numpy array or None.
-# ─────────────────────────────────────────────────────────────
-def decode_b64_frame(b64_str: str):
-    """Decode a base64 JPEG string to a BGR OpenCV image."""
-    try:
-        raw_bytes = base64.b64decode(b64_str)
-        np_arr    = np.frombuffer(raw_bytes, dtype=np.uint8)
-        frame     = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
-        return frame  # None if decode failed
-    except Exception as e:
-        print(f"⚠️  Frame decode error: {e}", flush=True)
-        return None
-
-
 # ─────────────────────────────────────────────────────────────
 #  LANDMARK UTILITIES
 # ─────────────────────────────────────────────────────────────
@@ -228,6 +186,7 @@ def pre_process_landmark(landmark_list):
 
 
 def apply_clahe(frame):
+    """Enhance low-light / low-contrast frames for mobile cameras."""
     try:
         lab     = cv2.cvtColor(frame, cv2.COLOR_BGR2LAB)
         l, a, b = cv2.split(lab)
@@ -238,25 +197,55 @@ def apply_clahe(frame):
 
 
 # ───────────────────────────────────────────────────��─────────
-#  EMOTION DETECTION v3
+#  EMOTION DETECTION — v3
+#  Redesigned SAD / ANGRY / SURPRISE with external facial cues.
+#  HAPPY unchanged.
 # ─────────────────────────────────────────────────────────────
+
 def _dist(a, b):
+    """Euclidean distance between two 2-D landmark points."""
     return float(np.linalg.norm(
         np.array(a, dtype=np.float32) - np.array(b, dtype=np.float32)
     ))
 
 
 def detect_emotion_from_landmarks(face_landmarks, img_shape):
+    """
+    Derive emotion scores from 468 MediaPipe Face Mesh landmarks.
+    Returns {emotion: float} summing to 1.0.
+
+    Key landmark reference:
+      Mouth corners : 61 (L), 291 (R)
+      Lip inner     : 13 (top), 14 (bottom)
+      Lip outer     : 0 (top), 17 (bottom)
+      L eye lids    : 159 (upper), 145 (lower)
+      R eye lids    : 386 (upper), 374 (lower)
+      L eye corners : 33 (outer), 133 (inner)
+      R eye corners : 362 (outer), 263 (inner)
+      L brow        : 70 (inner), 105 (outer), 52 (arch)
+      R brow        : 300 (inner), 334 (outer), 282 (arch)
+      Nose tip      : 4
+      Nostrils      : 129 (L ala), 358 (R ala)
+      Cheeks        : 50 (L), 280 (R)
+    """
     try:
         pts    = np.array(face_landmarks, dtype=np.float32)
         face_w = max(float(np.max(pts[:, 0]) - np.min(pts[:, 0])), 1.0)
         face_h = max(float(np.max(pts[:, 1]) - np.min(pts[:, 1])), 1.0)
 
-        def p(i):   return pts[i]
-        def nw(a,b): return _dist(p(a), p(b)) / face_w
-        def nh(a,b): return _dist(p(a), p(b)) / face_h
-        def clamp01(v): return max(0.0, min(1.0, float(v)))
+        def p(i):
+            return pts[i]
+
+        def nw(a, b):
+            return _dist(p(a), p(b)) / face_w
+
+        def nh(a, b):
+            return _dist(p(a), p(b)) / face_h
+
+        def clamp01(v):
+            return max(0.0, min(1.0, float(v)))
 
+        # ── Mouth ────────────────────────────────────────
         lc, rc = p(61),  p(291)
         tl, bl = p(13),  p(14)
         ut, lb = p(0),   p(17)
@@ -268,6 +257,7 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
         mouth_w     = _dist(lc, rc) / face_w
         o_ratio     = (mouth_open / (mouth_w + 1e-6)) if mouth_w > 0.05 else 0.0
 
+        # ── Eyes ─────────────────────────────────────────
         leu, led = p(159), p(145)
         reu, red = p(386), p(374)
         lel, ler = p(33),  p(133)
@@ -283,6 +273,7 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
         avg_ear = (l_ear + r_ear) / 2.0
         eye_open = ((l_eye_h + r_eye_h) / 2.0) / face_h
 
+        # ── Eyebrows ──────────────────────────────────────
         lib, rib = p(70),  p(300)
         lob, rob = p(105), p(334)
         lab, rab = p(52),  p(282)
@@ -302,13 +293,15 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
         r_arch_h   = (reu[1] - rab[1]) / face_h
         avg_arch_h = (l_arch_h + r_arch_h) / 2.0
 
+        # ── Nose ─────────────────────────────────────────
         nostril_w  = nw(129, 358)
         nose_flare = max(0.0, nostril_w - 0.26)
 
+        # ── Cheeks ───────────────────────────────────────
         lch, rch, nb = p(50), p(280), p(4)
         cheek_raise  = ((nb[1] - lch[1]) + (nb[1] - rch[1])) / (2.0 * face_h)
 
-        # HAPPY
+        # ── HAPPY ────────────────────────────────────────
         happy = 0.0
         if mouth_curve < -0.003:
             happy += clamp01(abs(mouth_curve) / 0.018) * 0.55
@@ -318,12 +311,12 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
             happy += clamp01(cheek_raise / 0.018) * 0.26
         if avg_ear < 0.27 and eye_open > 0.04:
             happy += 0.12
-        if mouth_curve > 0.004:   happy *= 0.15
-        if avg_brow_in_h < -0.05: happy *= 0.40
-        if mouth_open > 0.08:     happy *= 0.55
+        if mouth_curve > 0.004:         happy *= 0.15
+        if avg_brow_in_h < -0.05:       happy *= 0.40
+        if mouth_open > 0.08:           happy *= 0.55
         happy = clamp01(happy * 1.30)
 
-        # SAD v3
+        # ── SAD — v3 ──────────────────────────────────────
         sad = 0.0
         if brow_oblique > 0.02:
             sad += clamp01((brow_oblique - 0.02) / 0.055) * 0.45
@@ -338,13 +331,13 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
             sad += clamp01((0.36 - mouth_w) / 0.055) * 0.18
         if avg_brow_in_h > 0.04:
             sad += clamp01((avg_brow_in_h - 0.04) / 0.040) * 0.20
-        if mouth_curve < -0.004:                              sad *= 0.08
-        if cheek_raise > 0.014:                               sad *= 0.20
-        if avg_ear > 0.30:                                    sad *= 0.25
+        if mouth_curve < -0.004:                            sad *= 0.08
+        if cheek_raise > 0.014:                             sad *= 0.20
+        if avg_ear > 0.30:                                  sad *= 0.25
         if avg_brow_in_h < -0.06 and brow_inner_gap < 0.14: sad *= 0.15
         sad = clamp01(sad * 1.50)
 
-        # ANGRY v3
+        # ── ANGRY — v3 ────────────────────────────────────
         angry = 0.0
         if nose_flare > 0.005:
             angry += clamp01(nose_flare / 0.055) * 0.35
@@ -358,15 +351,16 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
             angry += clamp01((0.055 - eye_open) / 0.025) * 0.25
         if inner_gap < 0.030:
             angry += clamp01((0.030 - inner_gap) / 0.025) * 0.22
-        if brow_oblique < 0.015:  angry += 0.08
-        if mouth_curve < -0.005:  angry *= 0.15
-        if cheek_raise > 0.014:   angry *= 0.20
-        if avg_brow_in_h > -0.01: angry *= 0.20
-        if mouth_open > 0.07:     angry *= 0.40
-        if brow_oblique > 0.04:   angry *= 0.35
+        if brow_oblique < 0.015:
+            angry += 0.08
+        if mouth_curve < -0.005:        angry *= 0.15
+        if cheek_raise > 0.014:         angry *= 0.20
+        if avg_brow_in_h > -0.01:       angry *= 0.20
+        if mouth_open > 0.07:           angry *= 0.40
+        if brow_oblique > 0.04:         angry *= 0.35
         angry = clamp01(angry * 1.45)
 
-        # SURPRISE v3
+        # ── SURPRISE — v3 ─────────────────────────────────
         surprise = 0.0
         if 0.40 < o_ratio < 1.60:
             peak_dist = abs(o_ratio - 0.90)
@@ -383,16 +377,18 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
             surprise += clamp01((avg_arch_h - 0.05) / 0.055) * 0.35
         if avg_brow_in_h > 0.04 and avg_brow_out_h > 0.02:
             surprise += 0.15
-        if abs(brow_oblique) < 0.025: surprise += 0.08
-        if mouth_curve < -0.005:      surprise *= 0.30
-        if avg_brow_in_h < -0.03:    surprise *= 0.20
-        if avg_ear < 0.18:            surprise *= 0.25
-        if brow_inner_gap < 0.13:     surprise *= 0.25
-        if mouth_w > 0.44:            surprise *= 0.25
+        if abs(brow_oblique) < 0.025:
+            surprise += 0.08
+        if mouth_curve < -0.005:        surprise *= 0.30
+        if avg_brow_in_h < -0.03:       surprise *= 0.20
+        if avg_ear < 0.18:              surprise *= 0.25
+        if brow_inner_gap < 0.13:       surprise *= 0.25
+        if mouth_w > 0.44:              surprise *= 0.25
         if mouth_open > 0.12 and o_ratio < 0.40:
             surprise *= 0.30
         surprise = clamp01(surprise * 1.35)
 
+        # ── NEUTRAL ───────────────────────────────────────
         neutral = max(0.0, 1.0 - (happy + sad + angry + surprise) * 1.05)
 
         total = happy + sad + angry + surprise + neutral
@@ -407,12 +403,12 @@ def detect_emotion_from_landmarks(face_landmarks, img_shape):
                 for k, v in zip(EMOTIONS, [happy, sad, angry, surprise, neutral])}
 
     except Exception as ex:
-        print(f"⚠️  Emotion detection error: {ex}", flush=True)
+        print(f"⚠️  Emotion detection error: {ex}")
         return {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
 
 
 # ─────────────────────────────────────────────────────────────
-#  DISPLAY / SUGGESTION HELPERS
+#  DISPLAY HELPERS
 # ─────────────────────────────────────────────────────────────
 def get_word_suggestions(partial, word_freq):
     if not partial:
@@ -462,38 +458,49 @@ def draw_letter_overlay(frame, letter, confidence):
 #  CORE PROCESSING ENGINE
 # ─────────────────────────────────────────────────────────────
 def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, frame_seq=None):
-    print("=" * 70, flush=True)
-    print("🚀 ISL Detection Engine Starting…  (v4 — hardened frame pipeline)", flush=True)
-    print(f"   RUN_MODE       = {RUN_MODE}", flush=True)
-    print(f"   CONF_THRESHOLD = {CONF_THRESHOLD}", flush=True)
-    print(f"   LETTER_HOLD    = {LETTER_HOLD_SEC}s", flush=True)
-    print(f"   Thread         = {threading.current_thread().name}", flush=True)
-    print("=" * 70, flush=True)
-
-    # ── Load model ────────────────────────────────────────────
+    print("=" * 70)
+    print("🚀 ISL Detection System Starting…  (emotion engine v3)")
+    print(f"   RUN_MODE         = {RUN_MODE}")
+    print(f"   CONF_THRESHOLD   = {CONF_THRESHOLD}")
+    print(f"   LETTER_HOLD_SEC  = {LETTER_HOLD_SEC}s")
+    print(f"   SharedMemory     = {'enabled' if shm_name else 'disabled (base64 fallback)'}")
+    print("=" * 70)
+
+    # ── SharedMemory init ──────────────────────────────────────────────────
+    shm       = None
+    shm_array = None
+
+    if shm_name is not None:
+        try:
+            shm       = shared_memory.SharedMemory(name=shm_name)
+            shm_array = np.ndarray((480, 640, 3), dtype=np.uint8, buffer=shm.buf)
+            print("✅ SharedMemory connected (Zero-copy mode)")
+        except Exception as e:
+            print(f"❌ SharedMemory init failed: {e}")
+            return
+
+    # ── Load ISL model ─────────────────────────────────────────────────────
     try:
         model = load_model(ISL_MODEL_PATH)
-        print(f"✅ ISL model loaded: input={model.input_shape} output={model.output_shape}", flush=True)
+        print("✅ ISL model loaded")
+        print(f"   Input  shape : {model.input_shape}")
+        print(f"   Output shape : {model.output_shape}")
     except Exception as e:
-        print(f"❌ FATAL: Failed to load model: {e}", flush=True)
+        print(f"❌ Failed to load model: {e}")
+        if shm:
+            shm.close()
         return
 
-    # Validate model expects 42 features (21 landmarks × 2)
-    expected_features = model.input_shape[-1]
-    if expected_features != 42:
-        print(f"⚠️  Model expects {expected_features} features (expected 42). Results may be wrong.", flush=True)
-
     try:
-        model.predict(np.zeros((1, expected_features), np.float32), verbose=0)
-        print("✅ Model warmed up", flush=True)
+        model.predict(np.zeros((1, 42), np.float32), verbose=0)
+        print("✅ Model warmed up")
     except Exception as e:
-        print(f"⚠️  Warmup error: {e}", flush=True)
+        print(f"⚠️  Warmup: {e}")
 
     alphabet = list(string.ascii_uppercase) + [str(i) for i in range(1, 10)]
-    n_classes = model.output_shape[1]
-    print(f"   Alphabet: A-Z letters only  (model outputs {n_classes} classes, digits ignored)", flush=True)
+    print(f"   Alphabet : A-Z letters only  (model outputs {model.output_shape[1]} classes, digits ignored)")
 
-    # ── State ─────────────────────────────────────────────────
+    # ── Detection state ────────────────────────────────────────────────────
     current_word             = []
     sentence                 = []
     no_hand_frames           = 0
@@ -516,7 +523,6 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
 
     frame_count     = 0
     fps_frame_times = deque(maxlen=30)
-    last_log_time   = time.time()
 
     stats = {
         "letters_detected": 0,
@@ -526,40 +532,69 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
         "session_start":    time.time(),
     }
 
-    # ── Camera (LOCAL only) ───────────────────────────────────
-    cap = None
+    cmd_queue = shared_state.command_queue
+    ui_queue  = shared_state.ui_queue
+
+    # ── Camera init ────────────────────────────────────────────────────────
+    print("🎥 Initializing camera…")
     if RUN_MODE == "LOCAL":
-        print("🎥 Initializing local camera…", flush=True)
+        # FIX 2: cv2.CAP_DSHOW is Windows-only — use platform check
         if platform.system() == "Windows":
             cap = cv2.VideoCapture(CAM_INDEX, cv2.CAP_DSHOW)
         else:
-            cap = cv2.VideoCapture(CAM_INDEX)
+            cap = cv2.VideoCapture(CAM_INDEX)   # Linux/Mac — no CAP_DSHOW
+
         if not cap.isOpened():
-            print("❌ Camera not found", flush=True)
+            cap = cv2.VideoCapture(CAM_INDEX)   # fallback without backend flag
+        if not cap.isOpened():
+            print("❌ Camera not found — is another app using it?")
+            if shm:
+                shm.close()
             return
+
+        # FIX 4: Set MJPG codec BEFORE resolution — forces hardware MJPEG path
+        # on built-in laptop webcams, avoiding the raw uncompressed buffer that
+        # causes the horizontal static/noise stripes seen in the MJPEG stream.
+        # Must be set before width/height or it has no effect on some drivers.
         if platform.system() == "Windows":
-            cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M','J','P','G'))
+            cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))
+
         cap.set(cv2.CAP_PROP_FRAME_WIDTH,  CAMERA_RESOLUTION[0])
         cap.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_RESOLUTION[1])
         cap.set(cv2.CAP_PROP_FPS,          TARGET_FPS)
+
+        # FIX 5: Buffer size must be 1 to prevent stale frame accumulation.
+        # Set AFTER resolution so the driver doesn't reset it.
         cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
+
+        # FIX 4 continued: Flush 30 frames (was 10) to drain the DirectShow
+        # ring buffer. Built-in webcams pre-fill ~20 frames before stabilizing.
+        print("🎥 Flushing camera buffer (30 frames)…")
         for _ in range(30):
             cap.read()
-        ret, _ = cap.read()
-        if not ret:
-            print("❌ Camera flush failed", flush=True)
+
+        # Verify the camera is actually producing valid frames
+        ret, test_frame = cap.read()
+        if not ret or test_frame is None:
+            print("❌ Camera flush failed — no valid frame received")
             cap.release()
+            if shm:
+                shm.close()
             return
-        print("✅ Local camera ready", flush=True)
+
+        actual_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+        actual_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        print(f"✅ Camera: {actual_w}x{actual_h}")
+        print("✅ Camera ready!")
     else:
-        print("☁️  CLOUD mode — waiting for browser frames via socket", flush=True)
+        cap = None
+        print("☁️  CLOUD mode — waiting for browser frames via socket")
+
+    print("=" * 70)
 
-    # ── MediaPipe ─────────────────────────────────────────────
     mp_hands = mp.solutions.hands
     mp_face  = mp.solutions.face_mesh
 
-    print("⏳ Initializing MediaPipe…", flush=True)
-
     with mp_hands.Hands(
             static_image_mode=False,
             model_complexity=0,
@@ -575,82 +610,92 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
             min_tracking_confidence=0.6,
         ) as face_mesh:
 
-        print("✅ MediaPipe ready — entering main loop", flush=True)
+        print("✅ MediaPipe initialized — entering main loop")
 
         while True:
             t0  = time.time()
             now = t0
 
-            # ── Commands ──────────────────────────────────────
+            # ── Commands ───────────────────────────────────────────────────
             try:
-                while not shared_state.command_queue.empty():
-                    cmd    = shared_state.command_queue.get_nowait()
+                while not cmd_queue.empty():
+                    cmd    = cmd_queue.get_nowait()
                     action = cmd.get("action", "")
 
                     if action == "reset":
-                        current_word = []; sentence = []
-                        no_hand_frames = 0
-                        candidate_letter = ""; candidate_since = 0.0
-                        last_accepted_letter = ""; last_accepted_time = 0.0
+                        current_word         = []
+                        sentence             = []
+                        no_hand_frames       = 0
+                        candidate_letter     = ""
+                        candidate_since      = 0.0
+                        last_accepted_letter = ""
+                        last_accepted_time   = 0.0
                         letter_confidence_buffer.clear()
-                        current_detected_letter = ""; current_confidence = 0.0
-                        print("🔄 Reset", flush=True)
+                        current_detected_letter = ""
+                        current_confidence      = 0.0
+                        print("🔄 Reset")
 
                     elif action == "backspace":
-                        if current_word: current_word.pop()
-                        elif sentence:   sentence.pop()
+                        if current_word:
+                            current_word.pop()
+                        elif sentence:
+                            sentence.pop()
 
                     elif action == "accept_suggestion":
                         w = cmd.get("word", "")
-                        if w: current_word = list(w)
+                        if w:
+                            current_word = list(w)
 
                     elif action == "speak":
                         txt = (cmd.get("text", "")
                                or (" ".join(sentence) if sentence
                                    else "".join(current_word)))
-                        if txt: speak_text(txt)
+                        if txt:
+                            speak_text(txt)  # silently skipped in CLOUD mode
 
                     elif action == "stop":
-                        if cap: cap.release()
+                        if cap:
+                            cap.release()
+                        if shm:
+                            shm.close()
                         return
             except Exception:
                 pass
 
-            # ── Capture frame ─────────────────────────────────
-            raw = None
+            # ── Capture ────────────────────────────────────────────────────
             if RUN_MODE == "LOCAL":
                 ret, raw = cap.read()
                 if not ret or raw is None:
+                    print("⚠️  Camera read failed — retrying")
                     time.sleep(0.02)
                     continue
             else:
-                b64 = shared_state.take_frame()
-                if b64 is None:
+                fd = getattr(shared_state, "_cloud_frame", None)
+                if fd is None:
                     time.sleep(0.03)
                     continue
-                raw = decode_b64_frame(b64)
-                if raw is None:
-                    print("⚠️  Frame decode returned None — skipping", flush=True)
+                shared_state._cloud_frame = None
+                try:
+                    raw = cv2.imdecode(
+                        np.frombuffer(base64.b64decode(fd), np.uint8),
+                        cv2.IMREAD_COLOR)
+                    if raw is None:
+                        continue
+                except Exception:
                     continue
-                shared_state._frame_count_proc += 1
-
-            # ── Pre-process ───────────────────────────────────
-            try:
-                frame     = cv2.resize(raw, (FRAME_W, FRAME_H))
-                frame     = cv2.flip(frame, 1)
-                frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-            except Exception as e:
-                print(f"⚠️  Frame pre-process error: {e}", flush=True)
-                continue
 
+            frame     = cv2.resize(raw, (FRAME_W, FRAME_H))
+            frame     = cv2.flip(frame, 1)
+            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
             frame_rgb.flags.writeable = False
+
+            # ── Hand detection ─────────────────────────────────────────────
             hand_results = hands.process(frame_rgb)
             frame_rgb.flags.writeable = True
 
             detected_letter = ""
             hand_detected   = False
 
-            # ── Hand detection ────────────────────────────────
             if hand_results and hand_results.multi_hand_landmarks:
                 hand_detected  = True
                 no_hand_frames = 0
@@ -661,67 +706,59 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
                         lm_list  = calc_landmark_list(frame, hlm)
                         pre_list = pre_process_landmark(lm_list)
 
-                        if len(pre_list) != expected_features:
-                            print(f"⚠️  Landmark length {len(pre_list)} ≠ model expects {expected_features}", flush=True)
-                            continue
-
-                        features = np.array(pre_list, dtype=np.float32).reshape(1, expected_features)
-                        pred     = model.predict(features, verbose=0)
-                        prob     = float(np.max(pred))
-                        idx      = int(np.argmax(pred))
-                        idx      = min(idx, len(alphabet) - 1)
-                        raw_letter = alphabet[idx]
-
-                        # Skip number predictions — only A-Z letters
-                        if raw_letter.isdigit():
-                            current_detected_letter = ""
-                            current_confidence      = 0.0
-                            continue
-
-                        current_detected_letter = raw_letter
-                        current_confidence      = prob
-
-                        # Log every 30 frames so you can see detections in HF Logs
-                        if frame_count % 30 == 0:
-                            print(f"🔍 DEBUG → letter={raw_letter} conf={prob:.4f} "
-                                  f"frames_in={shared_state._frame_count_in} "
-                                  f"frames_proc={shared_state._frame_count_proc}", flush=True)
-
-                        if prob >= CONF_THRESHOLD:
-                            if raw_letter == candidate_letter:
-                                letter_confidence_buffer.append(prob)
+                        if model is not None and len(pre_list) == 42:
+                            features   = np.array(pre_list, dtype=np.float32).reshape(1, 42)
+                            pred       = model.predict(features, verbose=0)
+                            prob       = float(np.max(pred))
+                            raw_letter = alphabet[int(np.argmax(pred))]
+
+                            # Skip number predictions — only A-Z letters
+                            if raw_letter.isdigit():
+                                current_detected_letter = ""
+                                current_confidence      = 0.0
+                                continue
+
+                            current_detected_letter = raw_letter
+                            current_confidence      = prob
+
+                            print(f"DEBUG → {raw_letter} {prob:.4f}")
+
+                            if prob >= CONF_THRESHOLD:
+                                if raw_letter == candidate_letter:
+                                    letter_confidence_buffer.append(prob)
+                                else:
+                                    candidate_letter = raw_letter
+                                    candidate_since  = now
+                                    letter_confidence_buffer.clear()
+                                    letter_confidence_buffer.append(prob)
+
+                                avg_conf  = float(np.mean(letter_confidence_buffer))
+                                hold_time = now - candidate_since
+
+                                if hold_time >= LETTER_HOLD_SEC and avg_conf >= CONF_THRESHOLD:
+                                    same_letter_ok = (
+                                        raw_letter != last_accepted_letter or
+                                        (now - last_accepted_time) >= SAME_LETTER_COOLDOWN_SEC
+                                    )
+                                    if same_letter_ok:
+                                        detected_letter      = raw_letter
+                                        last_accepted_letter = raw_letter
+                                        last_accepted_time   = now
+                                        current_word.append(raw_letter)
+                                        stats["letters_detected"] += 1
+                                        print(f"✍️  Letter: {raw_letter}  "
+                                              f"(held {hold_time:.2f}s, conf {avg_conf:.3f})")
+                                        candidate_letter = ""
+                                        candidate_since  = 0.0
+                                        letter_confidence_buffer.clear()
                             else:
-                                candidate_letter = raw_letter
-                                candidate_since  = now
+                                candidate_letter = ""
+                                candidate_since  = 0.0
                                 letter_confidence_buffer.clear()
-                                letter_confidence_buffer.append(prob)
-
-                            avg_conf  = float(np.mean(letter_confidence_buffer))
-                            hold_time = now - candidate_since
-
-                            if hold_time >= LETTER_HOLD_SEC and avg_conf >= CONF_THRESHOLD:
-                                same_letter_ok = (
-                                    raw_letter != last_accepted_letter or
-                                    (now - last_accepted_time) >= SAME_LETTER_COOLDOWN_SEC
-                                )
-                                if same_letter_ok:
-                                    detected_letter      = raw_letter
-                                    last_accepted_letter = raw_letter
-                                    last_accepted_time   = now
-                                    current_word.append(raw_letter)
-                                    stats["letters_detected"] += 1
-                                    print(f"✍️  LETTER ACCEPTED: {raw_letter}  "
-                                          f"(held {hold_time:.2f}s, conf {avg_conf:.3f})", flush=True)
-                                    candidate_letter = ""
-                                    candidate_since  = 0.0
-                                    letter_confidence_buffer.clear()
-                        else:
-                            candidate_letter = ""
-                            candidate_since  = 0.0
-                            letter_confidence_buffer.clear()
 
                     except Exception as e:
-                        print(f"⚠️  Prediction error: {e}", flush=True)
+                        print(f"⚠️  Prediction error: {e}")
+
             else:
                 no_hand_frames += 1
                 letter_confidence_buffer.clear()
@@ -735,9 +772,9 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
                     sentence.append(word)
                     word_frequency[word] += 1
                     stats["words_formed"] += 1
-                    print(f"📝 Word committed: {word}", flush=True)
+                    print(f"📝 Word committed: {word}")
                     if ENABLE_AUTO_TTS:
-                        speak_text(word)
+                        speak_text(word)   # silently skipped in CLOUD mode
                     current_word         = []
                     last_accepted_letter = ""
                     last_accepted_time   = 0.0
@@ -745,7 +782,7 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
 
             frame = draw_letter_overlay(frame, current_detected_letter, current_confidence)
 
-            # ── Emotion (every N frames) ──────────────────────
+            # ── Emotion (every N frames) ───────────────────────────────────
             emotion_frame_counter += 1
             if emotion_frame_counter % EMOTION_PROCESS_INTERVAL == 0:
                 ns = {e: (1.0 if e == "neutral" else 0.0) for e in EMOTIONS}
@@ -758,10 +795,8 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
                         ns = detect_emotion_from_landmarks(
                             calc_landmark_list(frame, mr.multi_face_landmarks[0]),
                             frame.shape)
-                        if emotion_frame_counter % 50 == 0:
-                            print(f"😊 Emotion scores: { {k: f'{v:.3f}' for k,v in ns.items()} }", flush=True)
-                except Exception as e:
-                    print(f"⚠️  Emotion frame error: {e}", flush=True)
+                except Exception:
+                    pass
 
                 emotion_history.append(ns)
                 avg_e = defaultdict(float)
@@ -777,7 +812,11 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
 
                 if current_emotion != dominant:
                     stats["emotion_changes"] += 1
-                    print(f"😊 Emotion changed: {current_emotion} → {dominant}", flush=True)
+                    scores_str = " | ".join(
+                        [f"{k.upper()}:{avg_e[k]:.3f}" for k in EMOTIONS]
+                    )
+                    print(f"😊 Emotion: {current_emotion.upper()} → {dominant.upper()}")
+                    print(f"   Scores  : {scores_str}")
 
                 current_emotion = dominant
                 emotion_scores  = dict(avg_e)
@@ -787,7 +826,7 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
                     "scores":  dict(avg_e),
                 })
 
-            # ── FPS ───────────────────────────────────────────
+            # ── FPS ────────────────────────────────────────────────────────
             frame_count += 1
             fps_frame_times.append(time.time())
             if len(fps_frame_times) > 1:
@@ -795,27 +834,26 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
                 if td > 0:
                     shared_state.processing_fps.value = (len(fps_frame_times) - 1) / td
 
-            # ── Periodic heartbeat log ────────────────────────
-            now_t = time.time()
-            if now_t - last_log_time >= 10.0:
-                last_log_time = now_t
-                print(f"💓 Heartbeat | frame#{frame_count} | "
-                      f"fps={shared_state.processing_fps.value:.1f} | "
-                      f"in={shared_state._frame_count_in} | "
-                      f"proc={shared_state._frame_count_proc} | "
-                      f"emotion={current_emotion} | "
-                      f"word={''.join(current_word) or '(none)'}", flush=True)
-
-            # ── Build state packet (base64 frame for UI) ──────
-            try:
+            # ── Frame output ───────────────────────────────────────────────
+            if shm_array is not None:
+                shm_array[:] = frame
+                if frame_lock_val is not None:
+                    frame_lock_val.value = 1
+                if frame_seq is not None:
+                    frame_seq.value += 1
+                frame_base64 = None
+            else:
+                # PERF: Resize to 320×240 before encoding to halve payload size.
+                # 640×480 @ q65 ≈ 40KB; 320×240 @ q70 ≈ 12KB — 3× smaller.
+                # Browser renders it full-width so it still looks fine.
+                display = cv2.resize(frame, (320, 240))
                 _, buffer_img = cv2.imencode(
-                    ".jpg", frame, [cv2.IMWRITE_JPEG_QUALITY, 60])
+                    ".jpg", display, [cv2.IMWRITE_JPEG_QUALITY, 70])
                 frame_base64 = base64.b64encode(buffer_img.tobytes()).decode("utf-8")
-            except Exception:
-                frame_base64 = ""
 
+            # ── State packet ───────────────────────────────────────────────
             state_packet = {
-                "frame":              frame_base64,
+                "frame":              frame_base64 if frame_base64 else "",
                 "current_word":       "".join(current_word),
                 "sentence":           " ".join(sentence),
                 "suggestions":        get_word_suggestions(
@@ -837,29 +875,54 @@ def core_processing_engine(shared_state, shm_name=None, frame_lock_val=None, fra
             }
 
             try:
-                shared_state.ui_queue.put_nowait(state_packet)
-            except queue_module.Full:
+                ui_queue.put_nowait(state_packet)
+            except Exception:
                 try:
-                    shared_state.ui_queue.get_nowait()
-                    shared_state.ui_queue.put_nowait(state_packet)
+                    ui_queue.get_nowait()
+                    ui_queue.put_nowait(state_packet)
                 except Exception:
                     pass
 
             if frame_count == 1:
-                print("📤 First state packet sent to UI queue", flush=True)
+                print("📤 First frame sent — dashboard can read now")
+                if shm_array is not None:
+                    print("📤 SharedMemory active — zero-copy mode")
 
+            # ── Frame-rate limiter ─────────────────────────────────────────
             wait = (1.0 / TARGET_FPS) - (time.time() - t0)
             if wait > 0:
                 time.sleep(wait)
 
     if cap:
         cap.release()
-    print("✅ Detection engine stopped", flush=True)
+    if shm is not None:
+        shm.close()
+
+    print("✅ System stopped")
 
 
 # ─────────────────────────────────────────────────────────────
 #  STANDALONE ENTRY POINT
 # ─────────────────────────────────────────────────────────────
 if __name__ == "__main__":
-    ss = SharedState()
-    core_processing_engine(ss)
+    FRAME_BYTES = FRAME_W * FRAME_H * 3
+
+    # FIX 3: Clean up leftover shared memory from a previous crash
+    # Without this, restarting after a crash raises FileExistsError
+    try:
+        _old_shm = shared_memory.SharedMemory(name="isl_frame_shm")
+        _old_shm.close()
+        _old_shm.unlink()
+        print("🧹 Cleaned up leftover shared memory from previous run")
+    except FileNotFoundError:
+        pass  # normal — no leftover
+
+    shm  = shared_memory.SharedMemory(create=True, size=FRAME_BYTES, name="isl_frame_shm")
+    seq  = Value(ctypes.c_uint64, 0)
+    lock = Value(ctypes.c_bool, False)
+    ss   = SharedState()
+    try:
+        core_processing_engine(ss, shm_name=shm.name, frame_lock_val=lock, frame_seq=seq)
+    finally:
+        shm.close()
+        shm.unlink()

nginx.conf

@@ -23,6 +23,14 @@ http {
     uwsgi_temp_path       /tmp/nginx_uwsgi;
     scgi_temp_path        /tmp/nginx_scgi;
 
+    # ── KEY PERFORMANCE FIX ──────────────────────────────────────
+    # Keep camera frame POST bodies in RAM (not disk).
+    # Default 8KB caused every ~10KB base64 frame to be written to
+    # /tmp/nginx_client_body — adding huge disk I/O latency per frame.
+    client_body_buffer_size  512k;
+    client_max_body_size     5M;
+    # ────────────────────────────────────────────────────────────
+
     sendfile on;
     keepalive_timeout 65;
 

templates/dashboard.html

@@ -942,8 +942,12 @@
       });
 
       /* ── Browser Camera Streaming ──────────────────────────── */
+      // PERF: 160×120 @ quality 0.35 ≈ 6–8 KB per frame (was 320×240 @ 0.6 ≈ 50 KB).
+      // Sending at 4 fps (250 ms) avoids nginx disk-buffering warnings and the
+      // engineio "Too many packets in payload" error that was causing hard lag.
       let cameraStream = null;
       let videoElement = null;
+      let _frameInFlight = false;      // flow control — one frame at a time
       const captureCanvas = document.createElement('canvas');
       const captureCtx = captureCanvas.getContext('2d');
 
@@ -955,7 +959,7 @@
         }
         try {
           cameraStream = await navigator.mediaDevices.getUserMedia({
-            video: { width: { ideal: 640 }, height: { ideal: 480 }, facingMode: 'user' },
+            video: { width: { ideal: 320 }, height: { ideal: 240 }, facingMode: 'user' },
             audio: false
           });
           videoElement = document.createElement('video');
@@ -966,22 +970,27 @@
           videoElement.style.position = 'absolute';
           videoElement.style.top = '-10000px';
           videoElement.style.left = '-10000px';
-          videoElement.style.width = '320px';
-          videoElement.style.height = '240px';
+          videoElement.style.width = '160px';
+          videoElement.style.height = '120px';
           videoElement.style.pointerEvents = 'none';
           document.body.appendChild(videoElement);
 
           videoElement.addEventListener('loadeddata', () => {
-            captureCanvas.width = 320;
-            captureCanvas.height = 240;
+            captureCanvas.width  = 160;
+            captureCanvas.height = 120;
             if (!window.cameraInterval) {
               window.cameraInterval = setInterval(() => {
+                // Skip if a frame is still being processed by server
+                if (_frameInFlight) return;
                 if (document.visibilityState === 'visible' && socket && socket.connected) {
-                  captureCtx.drawImage(videoElement, 0, 0, 320, 240);
-                  const frame = captureCanvas.toDataURL('image/jpeg', 0.6);
+                  captureCtx.drawImage(videoElement, 0, 0, 160, 120);
+                  const frame = captureCanvas.toDataURL('image/jpeg', 0.35);
+                  _frameInFlight = true;
                   socket.emit('browser_frame', { frame: frame.split(',')[1] });
+                  // Auto-release lock after 500ms even if no ack received
+                  setTimeout(() => { _frameInFlight = false; }, 500);
                 }
-              }, 65); // ~15 FPS
+              }, 250); // 4 fps — smooth without overwhelming polling transport
             }
           });
         } catch (err) {
@@ -1033,6 +1042,8 @@
         });
 
         socket.on('state_update', (data) => {
+          // Release in-flight lock — server has processed our frame
+          _frameInFlight = false;
           try {
             if (data.frame) {
               document.getElementById('camera-feed').src = 'data:image/jpeg;base64,' + data.frame;