Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,13 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+app/assets/Hi.mp4 filter=lfs diff=lfs merge=lfs -text
+app/assets/I.mp4 filter=lfs diff=lfs merge=lfs -text
+app/assets/good[[:space:]]morning.mp4 filter=lfs diff=lfs merge=lfs -text
+app/assets/turn[[:space:]]right.mp4 filter=lfs diff=lfs merge=lfs -text
+app/models/hand_landmarker.task filter=lfs diff=lfs merge=lfs -text
+app/models/vosk-model-small-en-us-0.15/am/final.mdl filter=lfs diff=lfs merge=lfs -text
+app/models/vosk-model-small-en-us-0.15/graph/Gr.fst filter=lfs diff=lfs merge=lfs -text
+app/models/vosk-model-small-en-us-0.15/graph/HCLr.fst filter=lfs diff=lfs merge=lfs -text
+app/models/vosk-model-small-en-us-0.15/ivector/final.dubm filter=lfs diff=lfs merge=lfs -text
+app/models/vosk-model-small-en-us-0.15/ivector/final.ie filter=lfs diff=lfs merge=lfs -text

app/assets/Hi.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a086923e0b775fd7688fb6466f566d04b954ce606c3f25a90f1b53ac444a6f4
+size 102026

app/assets/I.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f3f61e6ef54066e7760b075579580924a347f9d13eaf8b0b7ac072bdc69dcd
+size 131776

app/assets/good morning.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07b8d2e11b88be7ddddbf312847d4ee76eb6905dcbe9b811baad1f13fe276a8b
+size 128671

app/assets/turn right.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ec79781d4f700af4218f4a0a68c38ccbdcdeca9dbbbcd68f43aba7c5b6e6bfb
+size 132573

app/main.py

@@ -0,0 +1,18 @@
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+from app.routes import sign_to_speech, speech_to_sign
+import os
+
+app = FastAPI(title="Sign-Speech Communication System API")
+
+# Mount assets directory to serve video files
+assets_path = os.path.join(os.path.dirname(__file__), "assets")
+app.mount("/assets", StaticFiles(directory=assets_path), name="assets")
+
+# Include routers
+app.include_router(sign_to_speech.router, tags=["Sign-to-Speech"])
+app.include_router(speech_to_sign.router, tags=["Speech-to-Sign"])
+
+@app.get("/")
+async def root():
+    return {"message": "Welcome to the Sign-Speech Communication System API"}

app/models/hand_landmarker.task

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbc2a30080c3c557093b5ddfc334698132eb341044ccee322ccf8bcf3607cde1
+size 7819105

app/models/labels.txt

@@ -0,0 +1,5 @@
+hello
+help
+no
+thanks
+yes

app/models/lstm_model.tflite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:501ab2516879ee305590d1a15b0fa569cdcd1a459708580fd81e7ce5a54cfc02
+size 333744

app/models/vosk-model-small-en-us-0.15/README

@@ -0,0 +1,9 @@
+US English model for mobile Vosk applications
+
+Copyright 2020 Alpha Cephei Inc
+
+Accuracy: 10.38 (tedlium test) 9.85 (librispeech test-clean)
+Speed: 0.11xRT (desktop)
+Latency: 0.15s (right context)
+
+

app/models/vosk-model-small-en-us-0.15/am/final.mdl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75370a0137f9daf8f469dedd7daa4513ae7a621f03240c6e512e2b50b656a7b6
+size 15962575

app/models/vosk-model-small-en-us-0.15/conf/mfcc.conf

@@ -0,0 +1,7 @@
+--sample-frequency=16000
+--use-energy=false
+--num-mel-bins=40
+--num-ceps=40
+--low-freq=20
+--high-freq=7600
+--allow-downsample=true

app/models/vosk-model-small-en-us-0.15/conf/model.conf

@@ -0,0 +1,10 @@
+--min-active=200
+--max-active=3000
+--beam=10.0
+--lattice-beam=2.0
+--acoustic-scale=1.0
+--frame-subsampling-factor=3
+--endpoint.silence-phones=1:2:3:4:5:6:7:8:9:10
+--endpoint.rule2.min-trailing-silence=0.5
+--endpoint.rule3.min-trailing-silence=0.75
+--endpoint.rule4.min-trailing-silence=1.0

app/models/vosk-model-small-en-us-0.15/graph/disambig_tid.int

@@ -0,0 +1,17 @@
+10015
+10016
+10017
+10018
+10019
+10020
+10021
+10022
+10023
+10024
+10025
+10026
+10027
+10028
+10029
+10030
+10031

app/models/vosk-model-small-en-us-0.15/graph/phones/word_boundary.int

@@ -0,0 +1,166 @@
+1 nonword
+2 begin
+3 end
+4 internal
+5 singleton
+6 nonword
+7 begin
+8 end
+9 internal
+10 singleton
+11 begin
+12 end
+13 internal
+14 singleton
+15 begin
+16 end
+17 internal
+18 singleton
+19 begin
+20 end
+21 internal
+22 singleton
+23 begin
+24 end
+25 internal
+26 singleton
+27 begin
+28 end
+29 internal
+30 singleton
+31 begin
+32 end
+33 internal
+34 singleton
+35 begin
+36 end
+37 internal
+38 singleton
+39 begin
+40 end
+41 internal
+42 singleton
+43 begin
+44 end
+45 internal
+46 singleton
+47 begin
+48 end
+49 internal
+50 singleton
+51 begin
+52 end
+53 internal
+54 singleton
+55 begin
+56 end
+57 internal
+58 singleton
+59 begin
+60 end
+61 internal
+62 singleton
+63 begin
+64 end
+65 internal
+66 singleton
+67 begin
+68 end
+69 internal
+70 singleton
+71 begin
+72 end
+73 internal
+74 singleton
+75 begin
+76 end
+77 internal
+78 singleton
+79 begin
+80 end
+81 internal
+82 singleton
+83 begin
+84 end
+85 internal
+86 singleton
+87 begin
+88 end
+89 internal
+90 singleton
+91 begin
+92 end
+93 internal
+94 singleton
+95 begin
+96 end
+97 internal
+98 singleton
+99 begin
+100 end
+101 internal
+102 singleton
+103 begin
+104 end
+105 internal
+106 singleton
+107 begin
+108 end
+109 internal
+110 singleton
+111 begin
+112 end
+113 internal
+114 singleton
+115 begin
+116 end
+117 internal
+118 singleton
+119 begin
+120 end
+121 internal
+122 singleton
+123 begin
+124 end
+125 internal
+126 singleton
+127 begin
+128 end
+129 internal
+130 singleton
+131 begin
+132 end
+133 internal
+134 singleton
+135 begin
+136 end
+137 internal
+138 singleton
+139 begin
+140 end
+141 internal
+142 singleton
+143 begin
+144 end
+145 internal
+146 singleton
+147 begin
+148 end
+149 internal
+150 singleton
+151 begin
+152 end
+153 internal
+154 singleton
+155 begin
+156 end
+157 internal
+158 singleton
+159 begin
+160 end
+161 internal
+162 singleton
+163 begin
+164 end
+165 internal
+166 singleton

app/models/vosk-model-small-en-us-0.15/ivector/final.dubm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8c5d7dd69d2122313baaf19f61f35dd3fa18b70c62ac0687e311e1c46e6daca7
+size 168048

app/models/vosk-model-small-en-us-0.15/ivector/final.ie

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f37faf90c375b9e4740b569398b5829ed9cc07d19be6d441f72c3b71d7efcc6
+size 8288887

app/models/vosk-model-small-en-us-0.15/ivector/global_cmvn.stats

@@ -0,0 +1,3 @@
+ [
+  1.682383e+11 -1.1595e+10 -1.521733e+10 4.32034e+09 -2.257938e+10 -1.969666e+10 -2.559265e+10 -1.535687e+10 -1.276854e+10 -4.494483e+09 -1.209085e+10 -5.64008e+09 -1.134847e+10 -3.419512e+09 -1.079542e+10 -4.145463e+09 -6.637486e+09 -1.11318e+09 -3.479773e+09 -1.245932e+08 -1.386961e+09 6.560655e+07 -2.436518e+08 -4.032432e+07 4.620046e+08 -7.714964e+07 9.551484e+08 -4.119761e+08 8.208582e+08 -7.117156e+08 7.457703e+08 -4.3106e+08 1.202726e+09 2.904036e+08 1.231931e+09 3.629848e+08 6.366939e+08 -4.586172e+08 -5.267629e+08 -3.507819e+08 1.679838e+09 
+  1.741141e+13 8.92488e+11 8.743834e+11 8.848896e+11 1.190313e+12 1.160279e+12 1.300066e+12 1.005678e+12 9.39335e+11 8.089614e+11 7.927041e+11 6.882427e+11 6.444235e+11 5.151451e+11 4.825723e+11 3.210106e+11 2.720254e+11 1.772539e+11 1.248102e+11 6.691599e+10 3.599804e+10 1.207574e+10 1.679301e+09 4.594778e+08 5.821614e+09 1.451758e+10 2.55803e+10 3.43277e+10 4.245286e+10 4.784859e+10 4.988591e+10 4.925451e+10 5.074584e+10 4.9557e+10 4.407876e+10 3.421443e+10 3.138606e+10 2.539716e+10 1.948134e+10 1.381167e+10 0 ]

app/models/vosk-model-small-en-us-0.15/ivector/online_cmvn.conf

@@ -0,0 +1 @@
+# configuration file for apply-cmvn-online, used in the script ../local/run_online_decoding.sh

app/models/vosk-model-small-en-us-0.15/ivector/splice.conf

@@ -0,0 +1,2 @@
+--left-context=3
+--right-context=3

app/routes/sign_to_speech.py

@@ -0,0 +1,40 @@
+from fastapi import APIRouter, UploadFile, File, HTTPException
+from app.services.hand_tracker_service import HandTracker
+from app.services.sign_recognition_service import GestureRecognizer
+import cv2
+import numpy as np
+import io
+from PIL import Image
+
+router = APIRouter()
+
+# Initialize services
+tracker = HandTracker()
+recognizer = GestureRecognizer()
+
+@router.post("/sign-to-speech")
+async def sign_to_speech(file: UploadFile = File(...)):
+    try:
+        # Read image
+        contents = await file.read()
+        nparr = np.frombuffer(contents, np.uint8)
+        img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+        
+        if img is None:
+            raise HTTPException(status_code=400, detail="Invalid image file")
+
+        # Detect hands
+        tracker.find_hands(img)
+        lm_list = tracker.get_landmark_data()
+        
+        if lm_list:
+            prediction = recognizer.process_landmarks(lm_list)
+            if prediction:
+                return prediction
+            return {"text": "Processing...", "confidence": 0.0}
+        else:
+            recognizer.clear()
+            return {"text": "No Hand Detected", "confidence": 0.0}
+            
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

app/routes/speech_to_sign.py

@@ -0,0 +1,26 @@
+from fastapi import APIRouter, UploadFile, File, HTTPException, Form
+from app.services.speech_recognition_service import SpeechRecognitionService
+from app.services.sign_generation_service import SignGenerationService
+
+router = APIRouter()
+
+# Initialize services
+speech_service = SpeechRecognitionService()
+sign_service = SignGenerationService()
+
+@router.post("/speech-to-sign")
+async def speech_to_sign(file: UploadFile = File(...), samplerate: int = Form(16000)):
+    try:
+        audio_data = await file.read()
+        text = speech_service.transcribe_full(audio_data, samplerate)
+        
+        if not text:
+            return {"text": "", "sign_sequence": []}
+            
+        sequence = sign_service.get_sign_sequence(text)
+        return {
+            "text": text,
+            "sign_sequence": sequence
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))

app/services/hand_tracker_service.py

@@ -0,0 +1,40 @@
+import cv2
+import numpy as np
+import mediapipe as mp
+from mediapipe.tasks import python
+from mediapipe.tasks.python import vision
+import os
+
+class HandTracker:
+    def __init__(self, max_hands=1, detection_con=0.5, track_con=0.5, model_path=None):
+        if model_path is None:
+            base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            model_path = os.path.join(base_dir, "models", "hand_landmarker.task")
+                
+        base_options = python.BaseOptions(model_asset_path=model_path)
+        options = vision.HandLandmarkerOptions(
+            base_options=base_options,
+            num_hands=max_hands,
+            min_hand_detection_confidence=detection_con,
+            min_hand_presence_confidence=track_con
+        )
+        self.detector = vision.HandLandmarker.create_from_options(options)
+        self.results = None
+
+    def find_hands(self, img):
+        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+        mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=img_rgb)
+        self.results = self.detector.detect(mp_image)
+        return self.results
+
+    def get_landmark_data(self):
+        if self.results and self.results.hand_landmarks:
+            my_hand = self.results.hand_landmarks[0]
+            lm_list = []
+            for lm in my_hand:
+                lm_list.extend([lm.x, lm.y, lm.z])
+            return lm_list
+        return None
+
+    def close(self):
+        self.detector.close()

app/services/sign_generation_service.py

@@ -0,0 +1,33 @@
+import os
+
+class SignGenerationService:
+    def __init__(self, assets_dir=None):
+        if assets_dir is None:
+            base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            assets_dir = os.path.join(base_dir, "assets")
+        self.assets_dir = assets_dir
+        self.assets_map = self._build_assets_map()
+
+    def _build_assets_map(self):
+        assets_map = {}
+        if not os.path.exists(self.assets_dir):
+            return assets_map
+            
+        for filename in os.listdir(self.assets_dir):
+            name = os.path.splitext(filename)[0].lower()
+            assets_map[name] = filename
+        return assets_map
+
+    def get_sign_sequence(self, text):
+        if not text:
+            return []
+            
+        words = text.lower().strip().split()
+        sequence = []
+        for word in words:
+            if word in self.assets_map:
+                sequence.append({
+                    "word": word,
+                    "asset": self.assets_map[word]
+                })
+        return sequence

app/services/sign_recognition_service.py

@@ -0,0 +1,83 @@
+import numpy as np
+import os
+
+try:
+    import tensorflow as tf
+    tflite = tf.lite
+except ImportError:
+    try:
+        import tflite_runtime.interpreter as tflite
+    except ImportError:
+        raise ImportError("Neither tensorflow nor tflite_runtime is installed.")
+
+class GestureRecognizer:
+    def __init__(self, model_path=None, label_path=None, threshold=0.8):
+        self.threshold = threshold
+        self.sequence_length = 30
+        self.sequence_buffer = []
+        
+        # Determine paths relative to this file
+        base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # app/
+        if model_path is None:
+            model_path = os.path.join(base_dir, "models", "lstm_model.tflite")
+        if label_path is None:
+            label_path = os.path.join(base_dir, "models", "labels.txt")
+
+        # Load Labels
+        self.labels = []
+        try:
+            with open(label_path, "r") as f:
+                self.labels = [line.strip() for line in f.readlines()]
+        except FileNotFoundError:
+            print(f"Warning: {label_path} not found.")
+            
+        # Load Model
+        try:
+            # We use the Interpreter class from whichever module we imported
+            if hasattr(tflite, 'Interpreter'):
+                self.interpreter = tflite.Interpreter(model_path=model_path)
+            else:
+                # Some versions of tflite_runtime have it at the top level
+                self.interpreter = tflite(model_path=model_path)
+            
+            self.interpreter.allocate_tensors()
+            self.input_details = self.interpreter.get_input_details()
+            self.output_details = self.interpreter.get_output_details()
+        except Exception as e:
+            print(f"Error loading LSTM model: {e}")
+            raise
+
+    def process_landmarks(self, landmarks):
+        lm_np = np.array(landmarks)
+        wrist = lm_np[:3]
+        relative_lm = lm_np - np.tile(wrist, 21)
+        
+        self.sequence_buffer.append(relative_lm)
+        if len(self.sequence_buffer) > self.sequence_length:
+            self.sequence_buffer.pop(0)
+            
+        if len(self.sequence_buffer) == self.sequence_length:
+            return self._predict()
+        return None
+
+    def _predict(self):
+        input_data = np.array([self.sequence_buffer], dtype=np.float32)
+        try:
+            self.interpreter.set_tensor(self.input_details[0]['index'], input_data)
+            self.interpreter.invoke()
+            output_data = self.interpreter.get_tensor(self.output_details[0]['index'])
+            prediction = np.squeeze(output_data)
+            max_index = np.argmax(prediction)
+            confidence = prediction[max_index]
+            
+            if confidence > self.threshold:
+                return {
+                    "text": self.labels[max_index] if self.labels else str(max_index),
+                    "confidence": float(confidence)
+                }
+        except Exception as e:
+            print(f"Inference error: {e}")
+        return None
+
+    def clear(self):
+        self.sequence_buffer = []

app/services/speech_recognition_service.py

@@ -0,0 +1,30 @@
+import os
+import json
+import vosk
+
+class SpeechRecognitionService:
+    def __init__(self, model_path=None):
+        if model_path is None:
+            base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+            model_path = os.path.join(base_dir, "models", "vosk-model-small-en-us-0.15")
+        
+        if not os.path.exists(model_path):
+             raise FileNotFoundError(f"Vosk model not found at {model_path}")
+
+        print(f"Loading Vosk model from {model_path}...")
+        self.model = vosk.Model(model_path)
+
+    def transcribe(self, audio_data, samplerate):
+        recognizer = vosk.KaldiRecognizer(self.model, samplerate)
+        if recognizer.AcceptWaveform(audio_data):
+            result = json.loads(recognizer.Result())
+            return result.get("text", "")
+        else:
+            partial = json.loads(recognizer.PartialResult())
+            return partial.get("partial", "")
+
+    def transcribe_full(self, audio_data, samplerate):
+        recognizer = vosk.KaldiRecognizer(self.model, samplerate)
+        recognizer.AcceptWaveform(audio_data)
+        result = json.loads(recognizer.FinalResult())
+        return result.get("text", "")