Initial commit with Hugging Face Docker and LFS configurations

.dockerignore

@@ -0,0 +1,21 @@
+# Git and local configs
+.git
+.gitignore
+.env
+README.md
+
+# Node modules and build output
+**/node_modules/
+**/dist/
+
+# Python environment and cache
+venv/
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Raw training data and duplicate models
+MP_Data/
+best_model.h5

.gitattributes

@@ -0,0 +1 @@
+*.h5 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,13 @@
+venv/
+.venv/
+node_modules/
+MP_Data/
+best_model.h5
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+.env
+dist/
+build/

Dockerfile

@@ -0,0 +1,48 @@
+# Stage 1: Build the React frontend
+FROM node:20-alpine AS frontend-builder
+WORKDIR /frontend
+
+# Copy dependency configs
+COPY frontend/gesture_words/package*.json ./
+
+# Install packages
+RUN npm install
+
+# Copy source code and build
+COPY frontend/gesture_words/ ./
+RUN npm run build
+
+# Stage 2: Build the Python backend
+FROM python:3.10-slim
+
+# Install system dependencies for MediaPipe / OpenCV
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 \
+    libglib2.0-0 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install Python requirements
+COPY backend/requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy backend files
+COPY backend/ /app/backend/
+COPY sign_language_model.h5 /app/backend/sign_language_model.h5
+
+# Copy built frontend assets
+COPY --from=frontend-builder /frontend/dist /app/dist
+
+# Expose port (Hugging Face Space default)
+ENV PORT=7860
+EXPOSE 7860
+
+# Set Hugging Face non-root user permissions
+RUN useradd -m -u 1000 user
+RUN chown -R user:user /app
+USER user
+
+# Run Flask application from backend directory
+WORKDIR /app/backend
+CMD ["python", "result.py"]

backend/requirements.txt

@@ -0,0 +1,6 @@
+tensorflow-cpu
+mediapipe
+flask
+flask-cors
+opencv-python-headless
+numpy

backend/result.py

@@ -0,0 +1,76 @@
+import os
+import numpy as np
+from flask import Flask, request, jsonify, send_from_directory
+from flask_cors import CORS
+from tensorflow.keras.models import load_model
+import base64
+import cv2
+import mediapipe as mp
+
+app = Flask(__name__, static_folder='../dist', static_url_path='/')
+CORS(app)  # Allow requests from frontend (e.g., React)
+
+@app.route('/')
+def serve():
+    return send_from_directory(app.static_folder, 'index.html')
+
+@app.route('/<path:path>')
+def serve_static(path):
+    if path != "" and os.path.exists(os.path.join(app.static_folder, path)):
+        return send_from_directory(app.static_folder, path)
+    else:
+        return send_from_directory(app.static_folder, 'index.html')
+
+# Load modp
+model = load_model('sign_language_model.h5')
+actions = np.array(['hi', 'me', 'thank you'])
+
+mp_hands = mp.solutions.hands
+hands = mp_hands.Hands(static_image_mode=False, max_num_hands=1, min_detection_confidence=0.7)
+
+@app.route('/predict', methods=['POST'])
+def predict():
+    data = request.get_json()
+    if not data or 'image' not in data:
+        return jsonify({'error': 'No image provided'}), 400
+
+    try:
+        # Decode base64 image
+        image_data = base64.b64decode(data['image'].split(',')[1])
+        nparr = np.frombuffer(image_data, np.uint8)
+        image = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+
+        # Hand detection
+        rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        results = hands.process(rgb)
+
+        if results.multi_hand_landmarks:
+            hand_landmarks = results.multi_hand_landmarks[0]
+            keypoints = []
+            for lm in hand_landmarks.landmark:
+                keypoints.extend([lm.x, lm.y, lm.z])
+
+            # Ensure consistent input shape: (30, 63)
+            sequence = np.array([keypoints] * 30)  # 30 identical frames
+            input_data = np.expand_dims(sequence, axis=0)
+            input_data = input_data / np.max(input_data + 1e-6)  # Avoid divide-by-zero
+
+            prediction = model.predict(input_data)[0]
+            confidence = float(np.max(prediction))
+            label = actions[np.argmax(prediction)]
+
+            return jsonify({
+                'prediction': label,
+                'confidence': round(confidence, 2)
+            }), 200
+        else:
+            return jsonify({'prediction': 'No hand detected', 'confidence': 0.0}), 200
+
+    except Exception as e:
+        return jsonify({'error': str(e)}), 500
+
+if __name__ == '__main__':
+    port = int(os.environ.get('PORT', 5000))
+    app.run(host='0.0.0.0', port=port, debug=True)
+
+

backend/sign.py

@@ -0,0 +1,69 @@
+import mediapipe as mp
+import cv2
+import numpy as np
+import os
+import time
+
+# Setup MediaPipe
+mp_hands = mp.solutions.hands
+hands = mp_hands.Hands(static_image_mode=False, max_num_hands=1, min_detection_confidence=0.5)
+mp_drawing = mp.solutions.drawing_utils
+
+# Create dataset directory
+DATA_PATH = os.path.join('MP_Data')
+actions = np.array(['hi', 'me', 'thank you'])  # Remove extra space
+no_sequences = 30
+sequence_length = 30
+
+# Create folders
+for action in actions:
+    for sequence in range(no_sequences):
+        os.makedirs(os.path.join(DATA_PATH, action, str(sequence)), exist_ok=True)
+
+# Start video capture
+cap = cv2.VideoCapture(0)
+for action in actions:
+    for sequence in range(no_sequences):
+        print(f"\nCollecting data for '{action}', sequence {sequence}")
+        time.sleep(2)  # Delay before collecting
+
+        for frame_num in range(sequence_length):
+            ret, frame = cap.read()
+            if not ret:
+                print("Failed to grab frame.")
+                continue
+
+            image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            results = hands.process(image)
+            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
+
+            if results.multi_hand_landmarks:
+                for hand_landmarks in results.multi_hand_landmarks:
+                    mp_drawing.draw_landmarks(image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
+
+                    # Extract keypoints
+                    keypoints = []
+                    for lm in hand_landmarks.landmark:
+                        keypoints.extend([lm.x, lm.y, lm.z])
+
+                    # Save keypoints
+                    npy_path = os.path.join(DATA_PATH, action, str(sequence), f"{frame_num}.npy")
+                    np.save(npy_path, keypoints)
+                    print(f"Saved: {npy_path}")
+
+            else:
+                print(f"No hand detected for frame {frame_num} in sequence {sequence}.")
+
+            # Add feedback to frame
+            cv2.putText(image, f'{action} - Seq {sequence}, Frame {frame_num}', (10, 30),
+                        cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 0, 0), 2)
+
+            cv2.imshow('Hand Tracking', image)
+            if cv2.waitKey(10) & 0xFF == ord('q'):
+                cap.release()
+                cv2.destroyAllWindows()
+                exit()
+
+cap.release()
+cv2.destroyAllWindows()
+

backend/tain.py

@@ -0,0 +1,61 @@
+import os
+import numpy as np
+from sklearn.model_selection import train_test_split
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense
+from tensorflow.keras.utils import to_categorical
+from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
+
+# ----- CONFIGURATION -----
+DATA_PATH = os.path.join('MP_Data')
+actions = np.array(['hi', 'me', 'thank you'])
+no_sequences = 30
+sequence_length = 30
+label_map = {label: num for num, label in enumerate(actions)}
+
+# ----- LOAD DATA -----
+sequences, labels = [], []
+
+for action in actions:
+    for sequence in range(no_sequences):
+        window = []
+        for frame_num in range(sequence_length):
+            try:
+                path = os.path.join(DATA_PATH, action, str(sequence), f"{frame_num}.npy")
+                res = np.load(path)
+                if res.shape == (63,):
+                    window.append(res)
+            except Exception as e:
+                print(f"Error loading {path}: {e}")
+        if len(window) == sequence_length:
+            sequences.append(window)
+            labels.append(label_map[action])
+
+X = np.array(sequences)
+y = to_categorical(labels).astype(int)
+
+# ----- NORMALIZE INPUT -----
+X = X / np.max(X)
+
+# ----- TRAIN TEST SPLIT -----
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
+
+# ----- BUILD MODEL -----
+model = Sequential()
+model.add(LSTM(64, return_sequences=True, activation='relu', input_shape=(30, 63)))
+model.add(LSTM(128, return_sequences=False, activation='relu'))
+model.add(Dense(64, activation='relu'))
+model.add(Dense(len(actions), activation='softmax'))
+
+model.compile(optimizer='Adam', loss='categorical_crossentropy', metrics=['accuracy'])
+
+# ----- CALLBACKS -----
+checkpoint = ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True)
+earlystop = EarlyStopping(monitor='val_loss', patience=5)
+
+# ----- TRAIN MODEL -----
+model.fit(X_train, y_train, epochs=50, validation_data=(X_test, y_test),
+          callbacks=[checkpoint, earlystop])
+
+# ----- SAVE FINAL MODEL -----
+model.save('sign_language_model.h5')

frontend/gesture_words/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

frontend/gesture_words/README.md

@@ -0,0 +1,12 @@
+# React + Vite
+
+This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules.
+
+Currently, two official plugins are available:
+
+- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) for Fast Refresh
+- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh
+
+## Expanding the ESLint configuration
+
+If you are developing a production application, we recommend using TypeScript with type-aware lint rules enabled. Check out the [TS template](https://github.com/vitejs/vite/tree/main/packages/create-vite/template-react-ts) for information on how to integrate TypeScript and [`typescript-eslint`](https://typescript-eslint.io) in your project.

frontend/gesture_words/eslint.config.js

@@ -0,0 +1,33 @@
+import js from '@eslint/js'
+import globals from 'globals'
+import reactHooks from 'eslint-plugin-react-hooks'
+import reactRefresh from 'eslint-plugin-react-refresh'
+
+export default [
+  { ignores: ['dist'] },
+  {
+    files: ['**/*.{js,jsx}'],
+    languageOptions: {
+      ecmaVersion: 2020,
+      globals: globals.browser,
+      parserOptions: {
+        ecmaVersion: 'latest',
+        ecmaFeatures: { jsx: true },
+        sourceType: 'module',
+      },
+    },
+    plugins: {
+      'react-hooks': reactHooks,
+      'react-refresh': reactRefresh,
+    },
+    rules: {
+      ...js.configs.recommended.rules,
+      ...reactHooks.configs.recommended.rules,
+      'no-unused-vars': ['error', { varsIgnorePattern: '^[A-Z_]' }],
+      'react-refresh/only-export-components': [
+        'warn',
+        { allowConstantExport: true },
+      ],
+    },
+  },
+]

frontend/gesture_words/index.html

@@ -0,0 +1,13 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Vite + React</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>

frontend/gesture_words/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "gesture_words",
+  "private": true,
+  "version": "0.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "lint": "eslint .",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@tailwindcss/vite": "^4.1.5",
+    "axios": "^1.9.0",
+    "react": "^19.0.0",
+    "react-dom": "^19.0.0",
+    "react-webcam": "^7.2.0",
+    "tailwindcss": "^4.1.5"
+  },
+  "devDependencies": {
+    "@eslint/js": "^9.22.0",
+    "@types/react": "^19.0.10",
+    "@types/react-dom": "^19.0.4",
+    "@vitejs/plugin-react": "^4.3.4",
+    "eslint": "^9.22.0",
+    "eslint-plugin-react-hooks": "^5.2.0",
+    "eslint-plugin-react-refresh": "^0.4.19",
+    "globals": "^16.0.0",
+    "vite": "^6.3.4"
+  }
+}

frontend/gesture_words/src/App.jsx

@@ -0,0 +1,149 @@
+import React, { useRef, useState, useEffect } from 'react';
+import Webcam from 'react-webcam';
+import axios from 'axios';
+
+const SignDetection = () => {
+  const webcamRef = useRef(null);
+  const [prediction, setPrediction] = useState('');
+  const [loading, setLoading] = useState(false);
+  const [history, setHistory] = useState([]);
+
+  useEffect(() => {
+    const interval = setInterval(async () => {
+      if (webcamRef.current && webcamRef.current.getScreenshot && !loading) {
+        const imageSrc = webcamRef.current.getScreenshot();
+        if (imageSrc) {
+          setLoading(true);
+          try {
+            const apiBase = import.meta.env.DEV ? 'http://localhost:5000' : '';
+            const response = await axios.post(`${apiBase}/predict`, {
+              image: imageSrc
+            });
+
+            const result = response.data.prediction;
+            setPrediction(result);
+            setHistory((prev) => [result, ...prev.slice(0, 4)]);
+
+            const utterance = new SpeechSynthesisUtterance(result);
+            speechSynthesis.speak(utterance);
+          } catch (error) {
+            console.error("Prediction failed", error);
+            setPrediction("Error predicting sign.");
+          }
+          setLoading(false);
+        }
+      }
+    }, 2000);
+
+    return () => clearInterval(interval);
+  }, [loading]);
+
+  return (
+    <div className="flex h-screen bg-gradient-to-br from-gray-100 to-gray-300">
+      
+      {/* Sidebar */}
+      <aside className="w-72 bg-white/80 backdrop-blur-lg shadow-2xl 
+                       px-8 py-10 hidden sm:flex flex-col rounded-r-3xl">
+        <h2 className="text-3xl font-extrabold text-gray-800 mb-12 tracking-wide">
+          Dashboard
+        </h2>
+
+        <nav className="flex flex-col gap-5 text-gray-700 font-medium text-lg">
+          {["Home", "Sign Detection", "Training Module", "History", "Settings"].map((item) => (
+            <a
+              key={item}
+              href="#"
+              className="hover:text-blue-600 hover:bg-blue-100 
+                         rounded-lg px-4 py-2 transition-all"
+            >
+              {item}
+            </a>
+          ))}
+        </nav>
+      </aside>
+
+      {/* Main Content */}
+      <div className="flex-1 flex flex-col">
+        
+        {/* Navbar */}
+        <header className="bg-white/80 backdrop-blur-md shadow-lg 
+                           px-10 py-6 flex items-center justify-between border-b">
+          <h1 className="text-3xl font-bold text-gray-800 tracking-wide">
+            GESTURE WORDS
+          </h1>
+
+          <button
+            className="bg-blue-600 text-white px-6 py-3 rounded-xl font-semibold
+                       shadow hover:bg-blue-700 transition-all text-lg"
+          >
+            Login
+          </button>
+        </header>
+
+        {/* Main Area */}
+        <main className="flex flex-col items-center justify-start flex-1 p-12 overflow-y-auto">
+
+          <div className="bg-white/70 backdrop-blur-xl rounded-3xl 
+                          shadow-2xl p-10 w-full max-w-4xl text-center
+                          ring-1 ring-gray-300">
+            
+            <h2 className="text-3xl font-semibold text-gray-700 mb-10 tracking-wide">
+              Live Sign Detection
+            </h2>
+
+            {/* Webcam Container */}
+            <div className="border-4 border-gray-300 rounded-2xl overflow-hidden shadow-xl 
+                            w-[480px] h-[360px] mx-auto mb-8">
+              <Webcam
+                audio={false}
+                ref={webcamRef}
+                screenshotFormat="image/jpeg"
+                width={480}
+                height={360}
+                className="rounded-2xl"
+              />
+            </div>
+
+            {/* Status */}
+            <p className="text-lg text-gray-600 mb-6 animate-pulse">
+              {loading ? "Analyzing..." : "Waiting for next frame..."}
+            </p>
+
+            {/* Prediction */}
+            {prediction && (
+              <div className="mt-4">
+                <h3 className="text-xl text-gray-600">Prediction:</h3>
+                <p className="text-blue-600 text-5xl font-extrabold drop-shadow-sm mt-2">
+                  {prediction}
+                </p>
+              </div>
+            )}
+
+            {/* History */}
+            {history.length > 0 && (
+              <div className="mt-10 text-left">
+                <h4 className="text-lg font-semibold text-gray-600 mb-3">
+                  Recent Predictions:
+                </h4>
+
+                <ul className="text-gray-700 text-lg space-y-3">
+                  {history.map((item, index) => (
+                    <li
+                      key={index}
+                      className="border py-3 px-4 rounded-xl bg-gray-50 shadow-sm"
+                    >
+                      {item}
+                    </li>
+                  ))}
+                </ul>
+              </div>
+            )}
+          </div>
+
+        </main>
+      </div>
+    </div>
+  );
+};
+
+export default SignDetection;

frontend/gesture_words/src/api.js

@@ -0,0 +1,4 @@
+await axios.post('http://localhost:5000/predict', {
+    image: imageSrc // base64 string
+  });
+  

frontend/gesture_words/src/index.css

@@ -0,0 +1 @@
+@import "tailwindcss";

frontend/gesture_words/src/main.jsx

@@ -0,0 +1,10 @@
+import { StrictMode } from 'react'
+import { createRoot } from 'react-dom/client'
+import './index.css'
+import App from './App.jsx'
+
+createRoot(document.getElementById('root')).render(
+  <StrictMode>
+    <App />
+  </StrictMode>,
+)

frontend/gesture_words/vite.config.js

@@ -0,0 +1,9 @@
+import { defineConfig } from 'vite'
+import react from '@vitejs/plugin-react'
+import tailwindcss from '@tailwindcss/vite'
+
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react(), tailwindcss()],
+})

sign_language_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c52bd12dc1182f0d52535edfa998f78d73f0d375df31e81a50820590ab264e06
+size 1724272