Instructions to use Manav2op/EMOTIA with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Manav2op/EMOTIA with Transformers:
# pip install -U transformers accelerate # Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("Manav2op/EMOTIA", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download frontend/advanced/AdvancedVideoAnalysis.js from Manav2op/EMOTIA: direct link, hf CLI and curl.
- Browser
- Download file 16.2 kB
-
https://huggingface.co/Manav2op/EMOTIA/resolve/main/frontend/advanced/AdvancedVideoAnalysis.js
- Command line
-
hf download hf://Manav2op/EMOTIA/frontend/advanced/AdvancedVideoAnalysis.js
-
curl -L -o AdvancedVideoAnalysis.js https://huggingface.co/Manav2op/EMOTIA/resolve/main/frontend/advanced/AdvancedVideoAnalysis.js
16.2 kB
| import { useState, useEffect, useRef, useCallback } from 'react'; | |
| import { motion, AnimatePresence } from 'framer-motion'; | |
| import { LineChart, Line, XAxis, YAxis, ResponsiveContainer, AreaChart, Area } from 'recharts'; | |
| import { Mic, MicOff, Video, VideoOff, Settings, Zap, Shield, BarChart3 } from 'lucide-react'; | |
| const AdvancedVideoAnalysis = () => { | |
| const [isAnalyzing, setIsAnalyzing] = useState(false); | |
| const [currentAnalysis, setCurrentAnalysis] = useState(null); | |
| const [analysisHistory, setAnalysisHistory] = useState([]); | |
| const [isConnected, setIsConnected] = useState(false); | |
| const [connectionQuality, setConnectionQuality] = useState('good'); | |
| const [modelVersion, setModelVersion] = useState('v2.0.0'); | |
| const [privacyMode, setPrivacyMode] = useState(false); | |
| const videoRef = useRef(null); | |
| const canvasRef = useRef(null); | |
| const wsRef = useRef(null); | |
| const streamRef = useRef(null); | |
| const sessionIdRef = useRef(`session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`); | |
| // WebRTC and WebSocket setup | |
| const initializeWebRTC = useCallback(async () => { | |
| try { | |
| const stream = await navigator.mediaDevices.getUserMedia({ | |
| video: { | |
| width: { ideal: 1280 }, | |
| height: { ideal: 720 }, | |
| frameRate: { ideal: 30 } | |
| }, | |
| audio: { | |
| sampleRate: 16000, | |
| channelCount: 1, | |
| echoCancellation: true, | |
| noiseSuppression: true | |
| } | |
| }); | |
| streamRef.current = stream; | |
| if (videoRef.current) { | |
| videoRef.current.srcObject = stream; | |
| } | |
| // Initialize WebSocket for real-time analysis | |
| initializeWebSocket(); | |
| setIsConnected(true); | |
| } catch (error) { | |
| console.error('WebRTC initialization failed:', error); | |
| setConnectionQuality('error'); | |
| } | |
| }, []); | |
| const initializeWebSocket = () => { | |
| const ws = new WebSocket(`ws://localhost:8000/ws/analyze/${sessionIdRef.current}`); | |
| ws.onopen = () => { | |
| console.log('WebSocket connected'); | |
| setConnectionQuality('good'); | |
| }; | |
| ws.onmessage = (event) => { | |
| const data = JSON.parse(event.data); | |
| if (data.error) { | |
| console.error('Analysis error:', data.error); | |
| setConnectionQuality('error'); | |
| } else { | |
| setCurrentAnalysis(data); | |
| setAnalysisHistory(prev => [...prev.slice(-99), data]); // Keep last 100 | |
| } | |
| }; | |
| ws.onclose = () => { | |
| console.log('WebSocket disconnected'); | |
| setIsConnected(false); | |
| setConnectionQuality('disconnected'); | |
| }; | |
| ws.onerror = (error) => { | |
| console.error('WebSocket error:', error); | |
| setConnectionQuality('error'); | |
| }; | |
| wsRef.current = ws; | |
| }; | |
| const startAnalysis = async () => { | |
| setIsAnalyzing(true); | |
| await initializeWebRTC(); | |
| }; | |
| const stopAnalysis = () => { | |
| setIsAnalyzing(false); | |
| // Stop WebRTC stream | |
| if (streamRef.current) { | |
| streamRef.current.getTracks().forEach(track => track.stop()); | |
| } | |
| // Close WebSocket | |
| if (wsRef.current) { | |
| wsRef.current.close(); | |
| } | |
| setIsConnected(false); | |
| setCurrentAnalysis(null); | |
| }; | |
| // Real-time frame capture and streaming | |
| useEffect(() => { | |
| if (!isAnalyzing || !videoRef.current || !wsRef.current) return; | |
| const captureFrame = () => { | |
| if (!isAnalyzing) return; | |
| const canvas = canvasRef.current; | |
| const video = videoRef.current; | |
| const ctx = canvas.getContext('2d'); | |
| if (video.videoWidth > 0 && video.videoHeight > 0) { | |
| canvas.width = video.videoWidth; | |
| canvas.height = video.videoHeight; | |
| ctx.drawImage(video, 0, 0); | |
| // Convert to blob and send via WebSocket | |
| canvas.toBlob((blob) => { | |
| if (blob && wsRef.current && wsRef.current.readyState === WebSocket.OPEN) { | |
| const reader = new FileReader(); | |
| reader.onload = () => { | |
| const data = { | |
| image: reader.result, | |
| timestamp: Date.now(), | |
| sessionId: sessionIdRef.current | |
| }; | |
| wsRef.current.send(JSON.stringify(data)); | |
| }; | |
| reader.readAsDataURL(blob); | |
| } | |
| }, 'image/jpeg', 0.8); | |
| } | |
| // Continue capturing at ~10 FPS | |
| setTimeout(captureFrame, 100); | |
| }; | |
| captureFrame(); | |
| return () => { | |
| // Cleanup | |
| }; | |
| }, [isAnalyzing]); | |
| // Connection quality monitoring | |
| useEffect(() => { | |
| const checkConnection = () => { | |
| if (wsRef.current) { | |
| const state = wsRef.current.readyState; | |
| if (state === WebSocket.CLOSED || state === WebSocket.CLOSING) { | |
| setConnectionQuality('disconnected'); | |
| } else if (state === WebSocket.CONNECTING) { | |
| setConnectionQuality('connecting'); | |
| } | |
| } | |
| }; | |
| const interval = setInterval(checkConnection, 1000); | |
| return () => clearInterval(interval); | |
| }, []); | |
| return ( | |
| <div className="min-h-screen bg-gradient-to-br from-gray-900 via-gray-800 to-gray-900 text-white"> | |
| {/* Advanced Header */} | |
| <header className="bg-black/20 backdrop-blur-xl border-b border-white/10 p-4"> | |
| <div className="max-w-7xl mx-auto flex justify-between items-center"> | |
| <div className="flex items-center space-x-4"> | |
| <motion.div | |
| initial={{ scale: 0 }} | |
| animate={{ scale: 1 }} | |
| className="text-3xl" | |
| > | |
| 🚀 | |
| </motion.div> | |
| <div> | |
| <h1 className="text-2xl font-bold bg-gradient-to-r from-cyan-400 to-violet-400 bg-clip-text text-transparent"> | |
| EMOTIA Advanced | |
| </h1> | |
| <p className="text-sm text-gray-400">Real-time Multi-Modal Intelligence</p> | |
| </div> | |
| </div> | |
| <div className="flex items-center space-x-4"> | |
| {/* Connection Status */} | |
| <div className="flex items-center space-x-2"> | |
| <div className={`w-3 h-3 rounded-full ${ | |
| connectionQuality === 'good' ? 'bg-green-400' : | |
| connectionQuality === 'connecting' ? 'bg-yellow-400 animate-pulse' : | |
| 'bg-red-400' | |
| }`} /> | |
| <span className="text-sm capitalize">{connectionQuality}</span> | |
| </div> | |
| {/* Model Version */} | |
| <div className="text-sm text-gray-400"> | |
| Model: {modelVersion} | |
| </div> | |
| {/* Privacy Mode */} | |
| <button | |
| onClick={() => setPrivacyMode(!privacyMode)} | |
| className={`p-2 rounded-lg transition-colors ${ | |
| privacyMode ? 'bg-red-600 hover:bg-red-700' : 'bg-gray-700 hover:bg-gray-600' | |
| }`} | |
| > | |
| <Shield className={`w-5 h-5 ${privacyMode ? 'text-white' : 'text-gray-400'}`} /> | |
| </button> | |
| {/* Control Buttons */} | |
| <motion.button | |
| whileHover={{ scale: 1.05 }} | |
| whileTap={{ scale: 0.95 }} | |
| onClick={isAnalyzing ? stopAnalysis : startAnalysis} | |
| className={`px-6 py-3 rounded-xl font-semibold transition-all duration-300 ${ | |
| isAnalyzing | |
| ? 'bg-gradient-to-r from-red-600 to-red-700 hover:from-red-700 hover:to-red-800 shadow-red-500/25' | |
| : 'bg-gradient-to-r from-cyan-600 to-violet-600 hover:from-cyan-700 hover:to-violet-700 shadow-cyan-500/25' | |
| } shadow-lg`} | |
| > | |
| <div className="flex items-center space-x-2"> | |
| <Zap className="w-5 h-5" /> | |
| <span>{isAnalyzing ? 'Stop Analysis' : 'Start Advanced Analysis'}</span> | |
| </div> | |
| </motion.button> | |
| </div> | |
| </div> | |
| </header> | |
| {/* Main Dashboard */} | |
| <main className="max-w-7xl mx-auto p-6 grid grid-cols-1 lg:grid-cols-12 gap-6"> | |
| {/* Video Feed Panel */} | |
| <div className="lg:col-span-4"> | |
| <motion.div | |
| initial={{ opacity: 0, y: 20 }} | |
| animate={{ opacity: 1, y: 0 }} | |
| className="bg-white/5 backdrop-blur-xl rounded-2xl p-6 border border-white/10" | |
| > | |
| <div className="flex items-center justify-between mb-4"> | |
| <h2 className="text-xl font-semibold text-cyan-400">Live Video Feed</h2> | |
| <div className="flex space-x-2"> | |
| <Video className="w-5 h-5 text-green-400" /> | |
| <Mic className="w-5 h-5 text-blue-400" /> | |
| </div> | |
| </div> | |
| <div className="relative aspect-video bg-black/50 rounded-xl overflow-hidden"> | |
| <video | |
| ref={videoRef} | |
| autoPlay | |
| muted | |
| className="w-full h-full object-cover" | |
| style={{ display: isAnalyzing ? 'block' : 'none' }} | |
| /> | |
| <canvas | |
| ref={canvasRef} | |
| className="w-full h-full" | |
| style={{ display: isAnalyzing ? 'none' : 'block' }} | |
| /> | |
| {!isAnalyzing && ( | |
| <div className="absolute inset-0 flex items-center justify-center"> | |
| <div className="text-center"> | |
| <div className="text-6xl mb-4">🎥</div> | |
| <p className="text-gray-400">Advanced analysis ready</p> | |
| <p className="text-sm text-gray-500 mt-2">WebRTC + WebSocket streaming</p> | |
| </div> | |
| </div> | |
| )} | |
| {/* Real-time overlay */} | |
| {isAnalyzing && currentAnalysis && ( | |
| <div className="absolute top-4 left-4 bg-black/70 backdrop-blur-sm rounded-lg p-3"> | |
| <div className="text-sm"> | |
| <div className="flex items-center space-x-2"> | |
| <div className="w-2 h-2 bg-green-400 rounded-full animate-pulse" /> | |
| <span>Processing: {currentAnalysis.processing_time?.toFixed(2)}s</span> | |
| </div> | |
| </div> | |
| </div> | |
| )} | |
| </div> | |
| </motion.div> | |
| </div> | |
| {/* Real-time Analytics */} | |
| <div className="lg:col-span-8 space-y-6"> | |
| {/* Emotion Timeline */} | |
| <motion.div | |
| initial={{ opacity: 0, x: 20 }} | |
| animate={{ opacity: 1, x: 0 }} | |
| className="bg-white/5 backdrop-blur-xl rounded-2xl p-6 border border-white/10" | |
| > | |
| <h2 className="text-xl font-semibold mb-4 text-lime-400">Real-time Emotion Timeline</h2> | |
| <div className="h-64"> | |
| <ResponsiveContainer width="100%" height="100%"> | |
| <AreaChart data={analysisHistory.slice(-20)}> | |
| <XAxis dataKey="timestamp" /> | |
| <YAxis domain={[0, 1]} /> | |
| <Area | |
| type="monotone" | |
| dataKey="engagement.mean" | |
| stroke="#10B981" | |
| fill="#10B981" | |
| fillOpacity={0.3} | |
| /> | |
| <Area | |
| type="monotone" | |
| dataKey="confidence.mean" | |
| stroke="#3B82F6" | |
| fill="#3B82F6" | |
| fillOpacity={0.3} | |
| /> | |
| </AreaChart> | |
| </ResponsiveContainer> | |
| </div> | |
| </motion.div> | |
| {/* Current Analysis */} | |
| <AnimatePresence> | |
| {currentAnalysis && ( | |
| <motion.div | |
| initial={{ opacity: 0, y: 20 }} | |
| animate={{ opacity: 1, y: 0 }} | |
| exit={{ opacity: 0, y: -20 }} | |
| className="grid grid-cols-1 md:grid-cols-2 gap-6" | |
| > | |
| {/* Emotion Analysis */} | |
| <div className="bg-white/5 backdrop-blur-xl rounded-2xl p-6 border border-white/10"> | |
| <h3 className="text-lg font-semibold mb-4 text-cyan-400">Emotion Analysis</h3> | |
| <div className="space-y-3"> | |
| {currentAnalysis.emotion?.probabilities?.map((prob, idx) => ( | |
| <div key={idx} className="flex items-center justify-between"> | |
| <span className="capitalize text-sm">{['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral'][idx]}</span> | |
| <div className="flex items-center space-x-2"> | |
| <div className="w-20 bg-gray-700 rounded-full h-2"> | |
| <motion.div | |
| initial={{ width: 0 }} | |
| animate={{ width: `${prob * 100}%` }} | |
| className="bg-gradient-to-r from-cyan-500 to-violet-500 h-2 rounded-full" | |
| /> | |
| </div> | |
| <span className="text-sm w-12 text-right">{(prob * 100).toFixed(1)}%</span> | |
| </div> | |
| </div> | |
| ))} | |
| </div> | |
| </div> | |
| {/* Intent Analysis */} | |
| <div className="bg-white/5 backdrop-blur-xl rounded-2xl p-6 border border-white/10"> | |
| <h3 className="text-lg font-semibold mb-4 text-violet-400">Intent Analysis</h3> | |
| <div className="space-y-3"> | |
| {currentAnalysis.intent?.probabilities?.map((prob, idx) => ( | |
| <div key={idx} className="flex items-center justify-between"> | |
| <span className="capitalize text-sm">{['agreement', 'confusion', 'hesitation', 'confidence', 'neutral'][idx]}</span> | |
| <div className="flex items-center space-x-2"> | |
| <div className="w-20 bg-gray-700 rounded-full h-2"> | |
| <motion.div | |
| initial={{ width: 0 }} | |
| animate={{ width: `${prob * 100}%` }} | |
| className="bg-gradient-to-r from-violet-500 to-pink-500 h-2 rounded-full" | |
| /> | |
| </div> | |
| <span className="text-sm w-12 text-right">{(prob * 100).toFixed(1)}%</span> | |
| </div> | |
| </div> | |
| ))} | |
| </div> | |
| </div> | |
| </motion.div> | |
| )} | |
| </AnimatePresence> | |
| {/* Modality Contributions */} | |
| {currentAnalysis?.modality_importance && ( | |
| <motion.div | |
| initial={{ opacity: 0, y: 20 }} | |
| animate={{ opacity: 1, y: 0 }} | |
| className="bg-white/5 backdrop-blur-xl rounded-2xl p-6 border border-white/10" | |
| > | |
| <h3 className="text-lg font-semibold mb-4 text-pink-400">AI Decision Factors</h3> | |
| <div className="grid grid-cols-3 gap-4"> | |
| {['Vision', 'Audio', 'Text'].map((modality, idx) => ( | |
| <div key={modality} className="text-center"> | |
| <div className="text-2xl mb-2"> | |
| {modality === 'Vision' ? '👁️' : modality === 'Audio' ? '🎵' : '💬'} | |
| </div> | |
| <div className="text-sm text-gray-400 mb-2">{modality}</div> | |
| <div className="text-xl font-bold text-pink-400"> | |
| {(currentAnalysis.modality_importance[idx] * 100).toFixed(1)}% | |
| </div> | |
| </div> | |
| ))} | |
| </div> | |
| </motion.div> | |
| )} | |
| </div> | |
| </main> | |
| {/* Footer */} | |
| <footer className="bg-black/20 backdrop-blur-xl border-t border-white/10 p-4 mt-8"> | |
| <div className="max-w-7xl mx-auto flex justify-between items-center text-sm text-gray-400"> | |
| <div>EMOTIA Advanced v2.0 - Real-time Multi-Modal Intelligence</div> | |
| <div className="flex items-center space-x-4"> | |
| <span>Privacy Mode: {privacyMode ? 'ON' : 'OFF'}</span> | |
| <span>WebRTC Active</span> | |
| <span>WebSocket Connected</span> | |
| </div> | |
| </div> | |
| </footer> | |
| </div> | |
| ); | |
| }; | |
| export default AdvancedVideoAnalysis; |