/** * SignLens — MediaPipe Landmark Neural Network Edition * * Runs classification directly in the browser using the weights * of our custom MLP Neural Network trained on the 21 hand landmarks. */ // ── Avatar 3D Module ──────────────────────────────────────────────────────── import { AvatarCanvas } from './avatar/AvatarCanvas.js?v=33'; let globalAvatar3D = null; window.globalAvatar3D = null; // ── Config ────────────────────────────────────────────────────────────────── let STREAK_NEEDED = 5; // frames of same letter to capture (settings sheet, matches native) const COOLDOWN_MS = 350; // ms after capture before detecting again (fluid sign transitions) let SPACE_FRAMES = 90; // no-hand frames before auto word push (~3.0s at 30fps, plenty of transition time) let MIN_CONFIDENCE = 0.50; // minimum classifier confidence (settings sheet, matches native 0.40-0.50) const MIN_HAND_SPAN = 35; // minimum pixel distance between wrist and middle MCP to reject ghost hands // ── MediaPipe Hand Connections ─────────────────────────────────────────────── const CONNECTIONS = [ [0,1],[1,2],[2,3],[3,4], [0,5],[5,6],[6,7],[7,8], [0,9],[9,10],[10,11],[11,12], [0,13],[13,14],[14,15],[15,16], [0,17],[17,18],[18,19],[19,20], [5,9],[9,13],[13,17], ]; const NODE_COLORS = { 0:'#ffffff', 1:'#f2856a',2:'#f2856a',3:'#f2856a',4:'#f2856a', 5:'#f0a53f',6:'#f0a53f',7:'#f0a53f',8:'#f0a53f', 9:'#6ec9a0',10:'#6ec9a0',11:'#6ec9a0',12:'#6ec9a0', 13:'#5aa9e6',14:'#5aa9e6',15:'#5aa9e6',16:'#5aa9e6', 17:'#b98ae0',18:'#b98ae0',19:'#b98ae0',20:'#b98ae0', }; const EDGE_COLORS = [ '#f2856a','#f2856a','#f2856a','#f2856a', // thumb 0-3 '#f0a53f','#f0a53f','#f0a53f','#f0a53f', // index 4-7 '#6ec9a0','#6ec9a0','#6ec9a0','#6ec9a0', // middle 8-11 '#5aa9e6','#5aa9e6','#5aa9e6','#5aa9e6', // ring 12-15 '#b98ae0','#b98ae0','#b98ae0','#b98ae0', // pinky 16-19 '#6a6a76','#6a6a76','#6a6a76', // palm 20-22 ]; // ── State ──────────────────────────────────────────────────────────────────── let currentMode = 'asl'; let paused = false; let inCooldown = false; let cameraActive = false; let currentWord = ''; let letterConfidences = []; let sentenceWords = []; let streakLetter = null; let streakCount = 0; let noHandCount = 0; // Camera view mirror state (default: false for normal real view, togglable to mirror) let cameraMirrorMode = localStorage.getItem('signlens_camera_mirror') === 'true'; // State tracking to prevent runaway letter repeat without releasing hand let lastCapturedLetter = ''; let releasedSinceLastCapture = true; let releaseFrames = 0; const RELEASE_FRAMES_NEEDED = 3; // State tracking to prevent runaway word repeat without clear user intent let lastCommittedWord = ''; // Trained Neural Network model weights (kept for OFFLINE fallback only) let aslWeights = null; let islWeights = null; let currentWeights = null; // ── Server-side inference state ────────────────────────────────────────────── // When Flask is running, all classification is done server-side (predict_proba // on the sklearn model) for feature parity with run_native.py. // If the server is unreachable, we fall back to the JS MLP. let serverAvailable = false; // set true after first successful /health probe let serverCheckPending = false; // ── DOM ────────────────────────────────────────────────────────────────────── const video = document.getElementById('webcam-video'); const displayCanvas = document.getElementById('display-canvas'); const dctx = displayCanvas.getContext('2d'); const noHandOverlay = document.getElementById('no-hand-overlay'); const detectedLetter = document.getElementById('detected-letter'); const detectedConf = document.getElementById('detected-conf'); const streakBar = document.getElementById('streak-bar'); const streakLabel = document.getElementById('streak-label'); const handBadge = document.getElementById('hand-badge'); const statusPill = document.getElementById('status-pill'); const statusText = document.getElementById('status-text'); const wordDisplay = document.getElementById('word-display'); const sentenceDisplay = document.getElementById('sentence-display'); const wordChips = document.getElementById('word-chips'); const modeToggle = document.getElementById('mode-toggle'); const toast = document.getElementById('toast'); // ── Utilities ──────────────────────────────────────────────────────────────── function showToast(msg, ms = 2000) { toast.textContent = msg; toast.classList.add('show'); setTimeout(() => toast.classList.remove('show'), ms); } function setStatus(cls, msg) { statusPill.className = `status-pill ${cls}`; statusText.textContent = msg; } function setStreakVisual(pct) { streakBar.style.width = `${pct}%`; // .dial-ring reads --streak as a registered and interpolates it. document.documentElement.style.setProperty('--streak', pct); } function resetStreak() { streakLetter = null; streakCount = 0; setStreakVisual(0); streakLabel.textContent = `0 / ${STREAK_NEEDED}`; } // ── Word / Sentence ─────────────────────────────────────────────────────────── function renderWord() { wordDisplay.innerHTML = currentWord ? currentWord : 'Start signing…'; } function renderSentence() { sentenceDisplay.innerHTML = sentenceWords.length ? sentenceWords.join(' ') : 'Your sentence will appear here…'; wordChips.innerHTML = ''; sentenceWords.forEach(w => { const c = document.createElement('span'); c.className = 'word-chip'; c.textContent = w; wordChips.appendChild(c); }); } async function pushWord() { let w = currentWord.trim(); if (w) { // Attempt spellcheck via backend API if available try { const res = await fetch('/spellcheck', { method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify({ word: w, confidences: letterConfidences }) }); if (res.ok) { const data = await res.json(); if (data.corrected) { w = data.corrected.toUpperCase(); } } } catch (e) { // Backend not present or offline, keep raw word } // Gating: Don't repeat the exact same word back-to-back if triggered by background jitter const isDuplicateAutoPush = (sentenceWords.length > 0 && sentenceWords[sentenceWords.length - 1] === w && lastCommittedWord === w); if (!isDuplicateAutoPush) { sentenceWords.push(w); lastCommittedWord = w; renderSentence(); showToast(`✓ "${w}"`); } } currentWord = ''; letterConfidences = []; renderWord(); resetStreak(); noHandCount = 0; lastCapturedLetter = ''; releasedSinceLastCapture = true; releaseFrames = 0; } // ── Letter capture ──────────────────────────────────────────────────────────── function captureLetter(letter, confidence = 0.85) { if (inCooldown) return; const l = letter.toLowerCase(); if (l === 'space' || l === '_') { pushWord(); return; } if (l === 'del' || l === 'delete') { currentWord = currentWord.slice(0, -1); letterConfidences.pop(); renderWord(); resetStreak(); lastCapturedLetter = ''; releasedSinceLastCapture = true; return; } if (l === 'nothing') return; currentWord += letter.toUpperCase(); letterConfidences.push(confidence); renderWord(); lastCapturedLetter = letter.toUpperCase(); releasedSinceLastCapture = false; releaseFrames = 0; detectedLetter.classList.remove('flash'); void detectedLetter.offsetWidth; detectedLetter.classList.add('flash'); resetStreak(); noHandCount = 0; inCooldown = true; setTimeout(() => { inCooldown = false; }, COOLDOWN_MS); } // ════════════════════════════════════════════════════════════════════════════ // ── Landmark Neural Network Classifier ────────────────────────────────────── // ════════════════════════════════════════════════════════════════════════════ /** 15 finger flexion triplets (vertex is the middle index) */ const ANGLE_TRIPLETS = [ [0,1,2], [1,2,3], [2,3,4], [0,5,6], [5,6,7], [6,7,8], [0,9,10], [9,10,11], [10,11,12], [0,13,14], [13,14,15], [14,15,16], [0,17,18], [17,18,19], [18,19,20], ]; /** Angle at vertex b formed by a-b-c, in radians normalized to [0,1] */ function calculateAngle3D(a, b, c) { const bax = a[0]-b[0], bay = a[1]-b[1], baz = a[2]-b[2]; const bcx = c[0]-b[0], bcy = c[1]-b[1], bcz = c[2]-b[2]; const dot = bax*bcx + bay*bcy + baz*bcz; const nba = Math.sqrt(bax*bax + bay*bay + baz*baz); const nbc = Math.sqrt(bcx*bcx + bcy*bcy + bcz*bcz); const cos = Math.min(1, Math.max(-1, dot / (nba*nbc + 1e-6))); return Math.acos(cos) / Math.PI; } /** * Builds the 78-D feature vector the MLP was trained on: * 63 aspect-corrected, wrist-centred, palm-scaled, rotation-aligned coords * + 15 finger flexion angles. * * Mirrors train_landmark_classifier.extract_features_from_image() exactly, * including its rotation convention. Do not "fix" the rotation here alone -- * the shipped weights were trained with it (see notes). */ function normalizeLandmarks(lm, w, h, handedness = "Right", applyHandednessMirror = false) { w = w || video.videoWidth || 640; h = h || video.videoHeight || 480; // Aspect-ratio corrected pixel space: (x*w, y*h, z*w) const pts = lm.map(p => [p.x * w, p.y * h, p.z * w]); // Mirror Left hand to match Right hand dataset distribution (ASL single hand) if (applyHandednessMirror && handedness === "Left") { const wristX = pts[0][0]; pts.forEach(p => { p[0] = wristX - (p[0] - wristX); }); } // Centre on the wrist const wrist = pts[0]; const cen = pts.map(p => [p[0]-wrist[0], p[1]-wrist[1], p[2]-wrist[2]]); // Palm scale = 2D wrist -> middle MCP distance let scale = Math.sqrt(cen[9][0]*cen[9][0] + cen[9][1]*cen[9][1]); if (scale < 1e-4) scale = 1; // 2D rotation keyed on landmark 9 against the negative Y axis const angle = Math.atan2(cen[9][0]/scale, -cen[9][1]/scale); const ca = Math.cos(angle), sa = Math.sin(angle); const features = []; // ISL uses the mathematically correct rotation matrix [[cos, -sin], [sin, cos]] // that aligns landmark 9 to vertical 12 o'clock, cancelling any hand tilt. const isIslMode = (typeof currentMode !== 'undefined' && currentMode === 'isl'); cen.forEach(p => { const x = p[0]/scale, y = p[1]/scale; if (isIslMode) { features.push(x*ca + y*sa); features.push(-x*sa + y*ca); } else { // ASL legacy rotation features.push(x*ca - y*sa); features.push(x*sa + y*ca); } features.push(p[2]/scale); // z is scaled but not rotated }); // Angles come from the raw pixel points, not the normalized ones ANGLE_TRIPLETS.forEach(([i, j, k]) => features.push(calculateAngle3D(pts[i], pts[j], pts[k])) ); return features; // 63 + 15 = 78 } /** Matrix multiplication helper */ function matMul(input, weights, bias) { const output = []; const numOutputs = bias.length; const numInputs = input.length; for (let j = 0; j < numOutputs; j++) { let sum = bias[j]; for (let i = 0; i < numInputs; i++) { sum += input[i] * weights[i][j]; } output.push(sum); } return output; } /** ReLU activation function */ function relu(arr) { return arr.map(x => Math.max(0, x)); } /** Softmax activation function */ function softmax(arr) { const max = Math.max(...arr); const exps = arr.map(x => Math.exp(x - max)); const sum = exps.reduce((a, b) => a + b, 0); return exps.map(x => x / sum); } /** * Probes /health to find out whether the Flask backend is up. * Called once at startup and retried whenever a classify() call fails. */ async function probeServer() { if (serverCheckPending) return; serverCheckPending = true; try { const r = await fetch('/health', { signal: AbortSignal.timeout(1500) }); if (r.ok) { if (!serverAvailable) { serverAvailable = true; console.log('✅ Flask backend detected — using server-side inference'); setStatus('ready', 'Connected'); } } else { serverAvailable = false; } } catch { serverAvailable = false; } finally { serverCheckPending = false; // If server is currently offline, automatically retry in 2.5s if (!serverAvailable && cameraActive) { setTimeout(probeServer, 2500); } } } /** Load MLP weights from JSON files (used for OFFLINE fallback only) */ async function loadLandmarkModel() { try { const resAsl = await fetch('./asl_landmarks_weights.json'); if (resAsl.ok) { aslWeights = await resAsl.json(); console.log('🧠 ASL Neural Network loaded (offline fallback)'); } const resIsl = await fetch('./isl_landmarks_weights.json'); if (resIsl.ok) { islWeights = await resIsl.json(); console.log('🧠 ISL Neural Network loaded (offline fallback)'); } currentWeights = currentMode === 'asl' ? aslWeights : islWeights; } catch (err) { console.error('Failed to load MLP weights:', err); } } /** * Assembles the 78-D (single-hand) or 156-D (dual-hand) feature vector. * * Sorting is on the RAW wrist x of the unflipped frame, identical to * run_native.extract_landmarks(). If these two ever disagree, every * two-handed sign silently decodes to the wrong letter. */ function buildFeatureVector(multiHandLandmarks, isDualMode, multiHandedness, w, h) { if (!isDualMode) { const handedness = (multiHandedness && multiHandedness[0] && multiHandedness[0].label) ? multiHandedness[0].label : 'Right'; // In ASL single hand mode, mirror Left hands to match Right hand dataset return normalizeLandmarks(multiHandLandmarks[0], w, h, handedness, true); } // Dual mode (ISL): // Single-hand ISL model is trained on BOTH Left & Right hands natively, // and two-handed classes strictly require both hands. // No fragile handedness guessing/mirroring is needed. const hands = multiHandLandmarks .map(lm => ({ x: lm[0].x, feat: normalizeLandmarks(lm, w, h, 'Right', false) })) .sort((a, b) => a.x - b.x); const slot1 = hands[0].feat; const slot2 = hands.length > 1 ? hands[1].feat : new Array(78).fill(0); return slot1.concat(slot2); } // Helper to check if hand landmarks represent a valid, non-ghost hand function isValidHand(lm) { if (!lm || lm.length < 21) return false; const w = displayCanvas.width || 640; const h = displayCanvas.height || 480; // Wrist (0) to Middle MCP (9) distance in pixels const dx = (lm[9].x - lm[0].x) * w; const dy = (lm[9].y - lm[0].y) * h; const span = Math.sqrt(dx * dx + dy * dy); // Reject tiny noise clusters / background artifacts if (span < MIN_HAND_SPAN) return false; // Check landmark coordinates are reasonably within frame [-0.2, 1.2] for (let i = 0; i < lm.length; i++) { if (isNaN(lm[i].x) || isNaN(lm[i].y)) return false; if (lm[i].x < -0.2 || lm[i].x > 1.2 || lm[i].y < -0.2 || lm[i].y > 1.2) return false; } return true; } /** * JS MLP fallback — used ONLY when Flask server is unreachable. * Runs the MLP forward pass in-browser using the cached weight JSON. */ function classifyJS(featureVector) { if (!currentWeights) return { letter: 'nothing', confidence: 0.0 }; const x = featureVector; if (x.length !== currentWeights.w0.length) { return { letter: 'nothing', confidence: 0.0 }; } let h = relu(matMul(x, currentWeights.w0, currentWeights.b0)); h = relu(matMul(h, currentWeights.w1, currentWeights.b1)); if (currentWeights.w2 && currentWeights.b2) { if (currentWeights.w3 && currentWeights.b3) { h = relu(matMul(h, currentWeights.w2, currentWeights.b2)); h = matMul(h, currentWeights.w3, currentWeights.b3); } else { h = matMul(h, currentWeights.w2, currentWeights.b2); } } let probs = softmax(h); // ── ISL Hand-Count Constrained Gating ───────────────────────────────── if (currentMode === 'isl' && x.length === 156) { const isOneHand = x.slice(78).every(v => Math.abs(v) < 1e-4); const islSingleHand = new Set(['1', '2', '3', '4', '5', '6', '7', '8', '9', 'C', 'I', 'J', 'L', 'O', 'Q', 'U', 'V']); for (let i = 0; i < probs.length; i++) { const cls = currentWeights.classes[i]; if (isOneHand ? !islSingleHand.has(cls) : islSingleHand.has(cls)) { probs[i] = 0; } } const pSum = probs.reduce((a, b) => a + b, 0); if (pSum > 0) { probs = probs.map(p => p / pSum); } } let maxIdx = 0, maxProb = 0, secondProb = 0; for (let i = 0; i < probs.length; i++) { if (probs[i] > maxProb) { secondProb = maxProb; maxProb = probs[i]; maxIdx = i; } else if (probs[i] > secondProb) { secondProb = probs[i]; } } const ranked = probs.map((p, i) => ({ label: currentWeights.classes[i], confidence: p })); ranked.sort((a, b) => b.confidence - a.confidence); const top3 = ranked.slice(0, 3); if ((maxProb - secondProb) < 0.08 && maxProb < 0.85) { return { letter: 'nothing', confidence: maxProb, top3, probs }; } return { letter: currentWeights.classes[maxIdx], confidence: maxProb, top3, probs }; } /** * PRIMARY classify() — async. * * Strategy: * 1. Build the feature vector in JS (normalization is correct and fast). * 2. POST it to Flask /predict_landmarks — uses sklearn's calibrated * predict_proba(), identical to run_native.py. This is why signs that * work in native Python also work in the browser. * 3. If the server is unavailable, fall back to the JS MLP. */ async function classify(multiHandLandmarks, multiHandedness) { if (!multiHandLandmarks || multiHandLandmarks.length === 0) { return { letter: 'nothing', confidence: 0.0 }; } // Determine if active model is dual-hand (156-D) or single-hand (78-D) const isDualMode = currentWeights ? currentWeights.w0.length === 156 : false; // Build feature vector with true display/camera aspect ratio const cw = (displayCanvas && displayCanvas.width) || (video && video.videoWidth) || 640; const ch = (displayCanvas && displayCanvas.height) || (video && video.videoHeight) || 480; const featureVector = buildFeatureVector(multiHandLandmarks, isDualMode, multiHandedness, cw, ch); // ── PATH A: Server-side inference (preferred) ──────────────────────────── if (serverAvailable) { try { const response = await fetch('/predict_landmarks', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: currentMode, landmarks: featureVector }), signal: AbortSignal.timeout(1500), // 1500ms resilient timeout per frame }); if (response.ok) { const data = await response.json(); return { letter: data.letter || 'nothing', confidence: data.confidence || 0.0, top3: data.top3 || [], }; } // Non-200 response — server up but something is wrong console.warn('[classify] Server error:', response.status); return { letter: 'nothing', confidence: 0.0 }; } catch (err) { // Timeout or network error → mark server down and re-probe console.warn('[classify] Server unreachable, switching to JS fallback:', err.name); serverAvailable = false; showToast('⚠️ Server offline — using JS fallback', 3000); setStatus('loading', 'JS fallback'); probeServer(); // async retry in background } } // ── PATH B: JS MLP fallback ────────────────────────────────────────────── if (!currentWeights) return { letter: 'nothing', confidence: 0.0 }; return classifyJS(featureVector); } // ── Process classification result ───────────────────────────────────────────── function processResult(letter, confidence) { const conf = Math.round(confidence * 100); detectedLetter.textContent = (letter === 'nothing' || confidence < MIN_CONFIDENCE) ? '' : letter.toUpperCase(); detectedConf.textContent = letter === 'nothing' ? 'Unclear gesture' : `${conf}% · ${letter.toUpperCase()}`; const effective = (confidence >= MIN_CONFIDENCE && letter !== 'nothing') ? letter : null; // Track release for repeating the same letter if (!effective || effective !== lastCapturedLetter) { releaseFrames++; if (releaseFrames >= RELEASE_FRAMES_NEEDED) { releasedSinceLastCapture = true; } } else { releaseFrames = 0; } if (paused || inCooldown || !effective) { if (!effective) resetStreak(); return; } // Prevent spamming the exact same letter repeatedly without releasing hand if (effective.toUpperCase() === lastCapturedLetter && !releasedSinceLastCapture) { resetStreak(); return; } noHandCount = 0; if (effective === streakLetter) { streakCount++; } else { streakLetter = effective; streakCount = 1; } const pct = Math.min(100, Math.round((streakCount / STREAK_NEEDED) * 100)); setStreakVisual(pct); streakLabel.textContent = `${Math.min(streakCount, STREAK_NEEDED)} / ${STREAK_NEEDED}`; if (streakCount >= STREAK_NEEDED) { captureLetter(streakLetter, confidence); } } // (handLandmarker and animFrameId declared near initMediaPipe below) // Prevent concurrent classify calls from piling up while a fetch is in flight let classifyInFlight = false; async function onResults(results) { dctx.save(); dctx.clearRect(0, 0, displayCanvas.width, displayCanvas.height); dctx.drawImage(results.image, 0, 0, displayCanvas.width, displayCanvas.height); // Filter for genuine hands with valid size and geometry const validHands = (results.multiHandLandmarks || []).filter(isValidHand); if (validHands.length > 0) { noHandOverlay.classList.remove('visible'); handBadge.className = 'hand-badge live'; if (currentMode === 'isl') { handBadge.textContent = validHands.length > 1 ? '2 HANDS · ISL (A-Z)' : '1 HAND · ISL (1-9, C, I, L, O, U, V)'; } else { handBadge.textContent = validHands.length > 1 ? '2 HANDS DETECTED' : 'HAND DETECTED'; } validHands.forEach(drawSkeleton); // Skip this frame if the previous classify() hasn't returned yet // (avoids a queue of stale requests building up on slow networks) if (!classifyInFlight) { classifyInFlight = true; try { const res = await classify(validHands, results.multiHandedness); processResult(res.letter, res.confidence); } finally { classifyInFlight = false; } } } else { noHandOverlay.classList.add('visible'); handBadge.className = 'hand-badge'; handBadge.textContent = 'NO HAND'; detectedLetter.textContent = ''; detectedConf.textContent = 'Waiting…'; resetStreak(); // Hand was dropped/removed -> immediately grant release permission for next letter releaseFrames++; if (releaseFrames >= RELEASE_FRAMES_NEEDED) { releasedSinceLastCapture = true; } if (currentWord) { noHandCount++; if (noHandCount >= SPACE_FRAMES) { pushWord(); } } } dctx.restore(); } function drawSkeleton(lm) { const w = displayCanvas.width; const h = displayCanvas.height; // Draw lines CONNECTIONS.forEach(([i, j], idx) => { const p1 = lm[i]; const p2 = lm[j]; dctx.beginPath(); dctx.moveTo(p1.x * w, p1.y * h); dctx.lineTo(p2.x * w, p2.y * h); dctx.strokeStyle = EDGE_COLORS[idx] || '#6a6a76'; // arrays are aligned; neutral if ever not dctx.lineWidth = 3; dctx.lineCap = 'round'; dctx.stroke(); }); // Draw points lm.forEach((p, i) => { const x = p.x * w; const y = p.y * h; dctx.beginPath(); dctx.arc(x, y, i === 0 ? 7 : 5, 0, 2 * Math.PI); dctx.fillStyle = NODE_COLORS[i] || '#ffffff'; dctx.fill(); dctx.strokeStyle = '#ffffff'; dctx.lineWidth = 1.5; dctx.stroke(); }); } /** * Renders a styled, in-theme camera failure inside the camera panel. * Replaces the native alert() MediaPipe would otherwise raise. */ function showCameraError(err) { const name = (err && err.name) ? err.name : 'Error'; const HINTS = { NotReadableError: 'Another application is already using the camera. Close it (video call, another browser tab, your editor\u2019s preview pane) and reload.', NotAllowedError: 'Camera permission was denied. Allow camera access for this site, then reload.', NotFoundError: 'No camera device was found on this machine.', NotSupportedError: 'This browser cannot access a camera on this page. A secure origin (https:// or localhost) is required.', OverconstrainedError: 'No camera matches the requested 640\u00d7480 video mode.', AbortError: 'The camera was released before it could start. Reload to retry.', }; const msg = HINTS[name] || (err && err.message) || 'The camera could not be started.'; noHandOverlay.innerHTML = '' + '
Camera unavailable
' + '
' + '
'; // textContent, not innerHTML -- the error string is not ours to trust as markup noHandOverlay.querySelector('.cam-error-msg').textContent = msg; noHandOverlay.querySelector('.cam-error-code').textContent = name; noHandOverlay.classList.add('error'); noHandOverlay.classList.remove('hidden'); noHandOverlay.setAttribute('role', 'alert'); setStatus('err', 'Camera unavailable'); detectedLetter.textContent = ''; detectedConf.textContent = 'No camera'; console.error('[SignLens] camera init failed:', err); } // ── MediaPipe Tasks Vision Shared Resolvers & Landmarkers ──────────────────── let visionResolver = null; async function getVisionResolver() { if (visionResolver) return visionResolver; const { FilesetResolver } = await import( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/vision_bundle.mjs' ); visionResolver = await FilesetResolver.forVisionTasks( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/wasm' ); return visionResolver; } // Standalone Static Image Landmarker (IMAGE mode, NO webcam needed!) let imageLandmarker = null; let imageLandmarkerLoading = null; async function getImageLandmarker() { if (imageLandmarker) return imageLandmarker; if (imageLandmarkerLoading) return imageLandmarkerLoading; imageLandmarkerLoading = (async () => { try { const { HandLandmarker } = await import( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/vision_bundle.mjs' ); const vision = await getVisionResolver(); try { imageLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task', delegate: 'GPU', }, runningMode: 'IMAGE', numHands: 2, minHandDetectionConfidence: 0.15, minHandPresenceConfidence: 0.15, minTrackingConfidence: 0.15, }); } catch (gpuErr) { console.warn('[ImageLandmarker] GPU delegate failed, falling back to CPU:', gpuErr); imageLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task', delegate: 'CPU', }, runningMode: 'IMAGE', numHands: 2, minHandDetectionConfidence: 0.15, minHandPresenceConfidence: 0.15, minTrackingConfidence: 0.15, }); } return imageLandmarker; } catch (err) { console.error('[ImageLandmarker] Initialization error:', err); imageLandmarkerLoading = null; return null; } })(); return imageLandmarkerLoading; } let handLandmarker = null; let animFrameId = null; /** * Converts new Tasks Vision HandLandmarker results into the same shape * the rest of app.js expects: * multiHandLandmarks: Array of [{x,y,z}, ...21] (same as old API) * multiHandedness: Array of {label: 'Right'|'Left'} (same field name) */ function adaptResults(result) { const rawLandmarks = result.landmarks || []; const rawHandedness = result.handedness || []; const multiHandLandmarks = []; const multiHandedness = []; for (let i = 0; i < rawLandmarks.length; i++) { const lms = rawLandmarks[i]; if (!lms || lms.length < 21) continue; // Anatomical validation: // Rejects facial false positives while fully supporting profile/curled hand signs like 'O', 'C', 'E', 'S' const palmLen = Math.hypot(lms[9].x - lms[0].x, lms[9].y - lms[0].y); const totalSpan = Math.max(...lms.map(p => Math.hypot(p.x - lms[0].x, p.y - lms[0].y))); if (palmLen < 0.03 || totalSpan < 0.06) { continue; } multiHandLandmarks.push(lms); const h = rawHandedness[i]; multiHandedness.push({ label: (h && h[0] && h[0].categoryName) ? h[0].categoryName : 'Right' }); } return { multiHandLandmarks, multiHandedness }; } async function initMediaPipe() { // Probe getUserMedia first so we can show a styled error instead of a // native alert() on failure. try { const probe = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480 } }); probe.getTracks().forEach(t => t.stop()); } catch (err) { showCameraError(err); return false; } // Dynamically import the Tasks Vision ES-module bundle. const { HandLandmarker } = await import( 'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@0.10.14/vision_bundle.mjs' ); const vision = await getVisionResolver(); handLandmarker = await HandLandmarker.createFromOptions(vision, { baseOptions: { modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task', delegate: 'GPU', }, runningMode: 'VIDEO', numHands: 2, minHandDetectionConfidence: 0.25, minHandPresenceConfidence: 0.25, minTrackingConfidence: 0.25, }); // Start camera stream let stream; try { stream = await navigator.mediaDevices.getUserMedia({ video: { width: 640, height: 480, facingMode: 'user' } }); } catch (err) { showCameraError(err); return false; } video.srcObject = stream; await new Promise(resolve => { video.onloadedmetadata = resolve; }); video.play(); cameraActive = true; setStatus('ready', 'Connected'); // Detection loop using requestAnimationFrame for smooth 30fps let lastVideoTime = -1; function detect() { if (!cameraActive) { animFrameId = null; return; } animFrameId = requestAnimationFrame(detect); if (video.currentTime === lastVideoTime) return; // no new frame yet lastVideoTime = video.currentTime; displayCanvas.width = video.videoWidth || 640; displayCanvas.height = video.videoHeight || 480; // detectForVideo uses the video timestamp in ms for temporal smoothing const result = handLandmarker.detectForVideo(video, performance.now()); const adapted = adaptResults(result); // Synthesise a results object matching what onResults() already expects onResults({ image: video, multiHandLandmarks: adapted.multiHandLandmarks, multiHandedness: adapted.multiHandedness, }); } detect(); return true; } // ── Controls Setup ──────────────────────────────────────────────────────────── function setupControls() { document.getElementById('btn-backspace').addEventListener('click', () => { currentWord = currentWord.slice(0, -1); letterConfidences.pop(); renderWord(); resetStreak(); }); document.getElementById('btn-add-space').addEventListener('click', () => { pushWord(); }); document.getElementById('btn-clear-word').addEventListener('click', () => { currentWord = ''; letterConfidences = []; renderWord(); resetStreak(); }); document.getElementById('btn-clear-sentence').addEventListener('click', () => { sentenceWords = []; currentWord = ''; letterConfidences = []; renderWord(); renderSentence(); resetStreak(); }); // Mirror Camera Toggle const btnMirror = document.getElementById('btn-mirror-toggle'); function updateMirrorUI() { if (!displayCanvas) return; if (cameraMirrorMode) { displayCanvas.classList.add('mirrored'); if (btnMirror) { btnMirror.innerHTML = '🪞 Mirror: On'; btnMirror.classList.add('active-btn'); } } else { displayCanvas.classList.remove('mirrored'); if (btnMirror) { btnMirror.innerHTML = '📷 Normal View'; btnMirror.classList.remove('active-btn'); } } } updateMirrorUI(); if (btnMirror) { btnMirror.addEventListener('click', () => { cameraMirrorMode = !cameraMirrorMode; localStorage.setItem('signlens_camera_mirror', cameraMirrorMode); updateMirrorUI(); showToast(cameraMirrorMode ? '🪞 Mirror Mode Enabled' : '📷 Normal View Enabled'); }); } document.getElementById('btn-pause-detection').addEventListener('click', function() { paused = !paused; this.textContent = paused ? 'Resume' : 'Pause'; this.classList.toggle('active-btn', paused); }); document.getElementById('btn-camera-toggle').addEventListener('click', function() { cameraActive = !cameraActive; this.textContent = cameraActive ? 'Stop Camera' : 'Start Camera'; this.classList.toggle('active-btn', !cameraActive); if (!cameraActive && animFrameId) { cancelAnimationFrame(animFrameId); animFrameId = null; } else if (cameraActive && !animFrameId && handLandmarker) { // restart loop let lastVideoTime = -1; function detect() { if (!cameraActive) { animFrameId = null; return; } animFrameId = requestAnimationFrame(detect); if (video.currentTime === lastVideoTime) return; lastVideoTime = video.currentTime; displayCanvas.width = video.videoWidth || 640; displayCanvas.height = video.videoHeight || 480; const result = handLandmarker.detectForVideo(video, performance.now()); const adapted = adaptResults(result); onResults({ image: video, multiHandLandmarks: adapted.multiHandLandmarks, multiHandedness: adapted.multiHandedness }); } detect(); } }); document.getElementById('btn-copy').addEventListener('click', () => { if (sentenceWords.length) { navigator.clipboard.writeText(sentenceWords.join(' ')); showToast('📋 Copied to clipboard!'); } }); // Global reference to prevent Chrome garbage-collection bug on SpeechSynthesisUtterance window._activeUtterance = null; function fallbackServerSpeak(text, btn) { fetch('/speak', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text }) }).then(res => res.json()) .then(data => { console.log('[SignLens] Server TTS spoke:', data); }) .catch(err => console.error('[SignLens] Server TTS failed:', err)) .finally(() => { if (btn) { btn.classList.remove('active'); btn.textContent = 'Speak'; } }); } function speakSentence() { const text = (sentenceWords.length ? sentenceWords.join(' ') : (sentenceDisplay.innerText || '').replace(/Your sentence will appear here[…\.]*/i, '') ).trim(); if (!text) { showToast('⚠️ No sentence to speak yet'); return; } const btnSpeak = document.getElementById('btn-speak'); if (btnSpeak) { btnSpeak.classList.add('active'); btnSpeak.textContent = 'Speaking…'; } showToast(`🔊 Speaking: "${text}"`); if ('speechSynthesis' in window) { try { window.speechSynthesis.cancel(); // Reset stuck Chromium speech queue const u = new SpeechSynthesisUtterance(text); window._activeUtterance = u; // Keep reference to prevent V8 GC drop u.lang = 'en-US'; u.rate = 0.95; u.pitch = 1.0; u.volume = 1.0; const voices = window.speechSynthesis.getVoices(); if (voices && voices.length > 0) { const enVoice = voices.find(v => v.lang.startsWith('en') && !v.localService) || voices.find(v => v.lang.startsWith('en')) || voices[0]; if (enVoice) u.voice = enVoice; } u.onend = () => { window._activeUtterance = null; if (btnSpeak) { btnSpeak.classList.remove('active'); btnSpeak.textContent = 'Speak'; } }; u.onerror = (e) => { console.warn('[SignLens] Browser speech error, falling back to server TTS:', e); window._activeUtterance = null; fallbackServerSpeak(text, btnSpeak); }; window.speechSynthesis.speak(u); if (window.speechSynthesis.paused) { window.speechSynthesis.resume(); } // Safety timeout to reset button state after utterance setTimeout(() => { if (btnSpeak && btnSpeak.textContent === 'Speaking…') { btnSpeak.classList.remove('active'); btnSpeak.textContent = 'Speak'; } }, Math.max(3000, text.split(' ').length * 1200)); } catch (err) { console.warn('[SignLens] SpeechSynthesis exception:', err); fallbackServerSpeak(text, btnSpeak); } } else { fallbackServerSpeak(text, btnSpeak); } } document.getElementById('btn-speak').addEventListener('click', speakSentence); // Mode buttons const btnAsl = document.getElementById('btn-asl'); const btnIsl = document.getElementById('btn-isl'); const btnVisually = document.getElementById('btn-visually'); const btnUpload = document.getElementById('btn-upload'); const liveWorkspace = document.getElementById('live-workspace'); const visuallyWorkspace = document.getElementById('visually-workspace'); const uploadWorkspace = document.getElementById('upload-workspace'); function hideAllWorkspaces() { if (liveWorkspace) liveWorkspace.classList.add('hidden'); if (visuallyWorkspace) visuallyWorkspace.classList.add('hidden'); if (uploadWorkspace) uploadWorkspace.classList.add('hidden'); btnAsl.classList.remove('active'); btnIsl.classList.remove('active'); if (btnVisually) btnVisually.classList.remove('active'); if (btnUpload) btnUpload.classList.remove('active'); modeToggle.classList.remove('isl-mode', 'visually-mode', 'upload-mode'); } btnAsl.addEventListener('click', () => { currentMode = 'asl'; currentWeights = aslWeights; hideAllWorkspaces(); btnAsl.classList.add('active'); if (liveWorkspace) liveWorkspace.classList.remove('hidden'); resetStreak(); showToast('Switched to ASL (Live Camera)'); }); btnIsl.addEventListener('click', () => { currentMode = 'isl'; currentWeights = islWeights; hideAllWorkspaces(); btnIsl.classList.add('active'); modeToggle.classList.add('isl-mode'); if (liveWorkspace) liveWorkspace.classList.remove('hidden'); resetStreak(); showToast('Switched to ISL (Live Camera)'); }); if (btnVisually) { btnVisually.addEventListener('click', () => { hideAllWorkspaces(); btnVisually.classList.add('active'); modeToggle.classList.add('visually-mode'); if (visuallyWorkspace) visuallyWorkspace.classList.remove('hidden'); activatePracticeStudio(); showToast('Practice Studio Active'); }); } if (btnUpload) { btnUpload.addEventListener('click', () => { hideAllWorkspaces(); btnUpload.classList.add('active'); modeToggle.classList.add('upload-mode'); if (uploadWorkspace) uploadWorkspace.classList.remove('hidden'); showToast('Upload Image Recognition Active'); }); } // Settings sheet. Both thresholds are read live by processResult(), // so no restart is needed after a change. const setConf = document.getElementById('set-confidence'); const outConf = document.getElementById('out-confidence'); const setStreak = document.getElementById('set-streak'); const outStreak = document.getElementById('out-streak'); const setTimeoutInput = document.getElementById('set-timeout'); const outTimeout = document.getElementById('out-timeout'); setConf.value = Math.round(MIN_CONFIDENCE * 100); outConf.value = `${setConf.value}%`; setStreak.value = STREAK_NEEDED; outStreak.value = STREAK_NEEDED; if (setTimeoutInput && outTimeout) { setTimeoutInput.value = (SPACE_FRAMES / 30).toFixed(1); outTimeout.value = `${setTimeoutInput.value}s`; } setConf.addEventListener('input', () => { MIN_CONFIDENCE = Number(setConf.value) / 100; outConf.value = `${setConf.value}%`; }); setStreak.addEventListener('input', () => { STREAK_NEEDED = Number(setStreak.value); outStreak.value = STREAK_NEEDED; resetStreak(); // rescale the dial against the new target }); if (setTimeoutInput && outTimeout) { setTimeoutInput.addEventListener('input', () => { const sec = Number(setTimeoutInput.value); SPACE_FRAMES = Math.round(sec * 30); outTimeout.value = `${sec.toFixed(1)}s`; }); } // Theme Toggle (Light / Dark Mode) const btnThemeToggle = document.getElementById('btn-theme-toggle'); const iconThemeToggle = document.getElementById('theme-toggle-icon'); function applyTheme(theme) { const isLight = theme === 'light'; document.documentElement.setAttribute('data-theme', isLight ? 'light' : 'dark'); localStorage.setItem('signlens_theme', isLight ? 'light' : 'dark'); if (iconThemeToggle) { iconThemeToggle.textContent = isLight ? '🌙' : '☀️'; } if (btnThemeToggle) { btnThemeToggle.setAttribute('title', isLight ? 'Switch to Dark Mode' : 'Switch to Light Mode'); btnThemeToggle.setAttribute('aria-label', isLight ? 'Switch to Dark Mode' : 'Switch to Light Mode'); } // Re-render avatar if in practice mode so canvas background updates immediately if (typeof drawAvatar === 'function') { try { drawAvatar(); } catch (_) {} } } // Load saved preference or fallback to dark const savedTheme = localStorage.getItem('signlens_theme') || 'dark'; applyTheme(savedTheme); if (btnThemeToggle) { btnThemeToggle.addEventListener('click', () => { const currentTheme = document.documentElement.getAttribute('data-theme') || 'dark'; const nextTheme = currentTheme === 'light' ? 'dark' : 'light'; applyTheme(nextTheme); showToast(nextTheme === 'light' ? '☀️ Switched to Light Mode' : '🌙 Switched to Dark Mode'); }); } // Keyboard shortcuts document.addEventListener('keydown', (e) => { // Ignore shortcuts while a control has focus, or a sheet is open. if (e.target.closest('input, textarea, [popover]')) return; if (e.key === ' ' || e.code === 'Space') { e.preventDefault(); pushWord(); } else if (e.key === 'Backspace') { currentWord = currentWord.slice(0, -1); letterConfidences.pop(); renderWord(); } else if (e.key === '1') { btnAsl.click(); } else if (e.key === '2') { btnIsl.click(); } else if (e.key === '3') { if (btnVisually) btnVisually.click(); } else if (e.key === '4') { if (btnUpload) btnUpload.click(); } }); } // ── Sign Reference Guide Modal ─────────────────────────────────────────────── // ── Sign Reference Guide Modal ─────────────────────────────────────────────── function initSignGuide() { const panelGuide = document.getElementById('panel-guide'); const btnOpenGuide = document.getElementById('btn-open-guide'); const btnCloseGuide = document.getElementById('btn-close-guide'); const tabIsl = document.getElementById('tab-guide-isl'); const tabAsl = document.getElementById('tab-guide-asl'); const contentIsl = document.getElementById('guide-content-isl'); const contentAsl = document.getElementById('guide-content-asl'); const singleGrid = document.getElementById('isl-single-cards'); const dualGrid = document.getElementById('isl-dual-cards'); const islDescFallback = { '1': 'Index finger extended upwards', '2': 'Index and middle fingers extended in V-shape', '3': 'Thumb, index, and middle fingers extended', '4': 'Four fingers extended upwards', '5': 'Open hand with all five fingers spread', '6': 'Thumb and little finger extended', '7': 'Thumb, index, and little finger extended', '8': 'Middle, ring, and little fingers extended', '9': 'Thumb touching index finger in circle', 'C': 'Curved hand forming C-shape', 'I': 'Little finger extended upwards', 'L': 'Thumb and index finger forming L-shape', 'O': 'All fingers curved touching thumb in O-shape', 'U': 'Index and middle fingers held together vertically', 'V': 'Index and middle fingers in open V-shape', 'A': 'Dominant index finger touching non-dominant thumb tip', 'B': 'Both hands forming double circles / glasses pose', 'D': 'Dominant index pointing to non-dominant index forming D', 'E': 'Left hand curved C/horizontal, right index touching middle knuckle', 'F': 'Dominant index and middle crossing non-dominant index and middle', 'G': 'Both fists held together vertically with knuckles facing forward', 'H': 'Dominant open palm sweeping flat across non-dominant palm', 'J': 'Index finger drawing J stroke across non-dominant palm', 'K': 'Dominant index finger hooking over non-dominant index finger', 'M': 'Dominant three fingers laid across non-dominant palm', 'N': 'Dominant two fingers laid across non-dominant palm', 'P': 'Dominant index finger forming circle with thumb touching non-dominant tip', 'Q': 'Dominant hand forming circle loop with non-dominant index through', 'R': 'Dominant index finger hooked on non-dominant palm', 'S': 'Dominant pinky hooking non-dominant pinky / fist lock', 'T': 'Dominant index finger placed perpendicularly on non-dominant index side', 'W': 'Interlocking open fingers of both hands upright', 'X': 'Crossing both index fingers in an X', 'Y': 'Dominant index pointing between non-dominant thumb and index', 'Z': 'Dominant open palm held upright against non-dominant palm' }; function createGuideCard(letter, isTwoHanded, desc) { const el = document.createElement('div'); el.className = 'guide-card'; el.title = `${letter}: ${desc} (click to enlarge)`; el.dataset.letter = letter; el.dataset.hands = isTwoHanded ? '2' : '1'; el.dataset.desc = desc; el.dataset.lang = 'ISL'; el.dataset.imgSrc = `isl_signs/${letter}.png`; el.innerHTML = `
Sign ${letter}
${letter} ${isTwoHanded ? '2 Hands' : '1 Hand'}

${desc}

`; el.addEventListener('click', () => openSignLightbox(el)); return el; } function renderGuideCards() { if (!singleGrid || !dualGrid) return; singleGrid.innerHTML = ''; dualGrid.innerHTML = ''; const isl = (practiceTemplates && practiceTemplates.isl) ? practiceTemplates.isl : {}; const singleKeys = ['1', '2', '3', '4', '5', '6', '7', '8', '9', 'C', 'I', 'L', 'O', 'U', 'V']; const dualKeys = ['A', 'B', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'W', 'X', 'Y', 'Z']; singleKeys.forEach(k => { const desc = (isl[k] && isl[k].desc) ? isl[k].desc : (islDescFallback[k] || 'Single hand sign'); singleGrid.appendChild(createGuideCard(k, false, desc)); }); dualKeys.forEach(k => { const desc = (isl[k] && isl[k].desc) ? isl[k].desc : (islDescFallback[k] || 'Dual hand sign'); dualGrid.appendChild(createGuideCard(k, true, desc)); }); } const aslDescFallback = { 'A': 'Fist with thumb on side', 'B': 'All fingers together pointing up', 'C': 'Curved hand in C-shape', 'D': 'Index up, fingers curve to thumb', 'E': 'Fingers bent, thumb tucked', 'F': 'Index-thumb circle, three fingers up', 'G': 'Index and thumb point sideways', 'H': 'Two fingers point sideways', 'I': 'Pinky finger up', 'J': 'Pinky draws J in air', 'K': 'Index up, middle angled, thumb out', 'L': 'L-shape — thumb and index', 'M': 'Three fingers over thumb', 'N': 'Two fingers over thumb', 'O': 'All fingers curve to form O', 'P': 'K shape pointing down', 'Q': 'G shape pointing down', 'R': 'Two fingers crossed', 'S': 'Fist with thumb over fingers', 'T': 'Thumb between index and middle', 'U': 'Two fingers together pointing up', 'V': 'Two fingers spread in V', 'W': 'Three fingers spread out', 'X': 'Index finger hooks', 'Y': 'Thumb and pinky out', 'Z': 'Index draws Z in air' }; function renderAslCards() { const aslGrid = document.getElementById('asl-cards'); if (!aslGrid || aslGrid.children.length > 0) return; // render once aslGrid.innerHTML = ''; const aslLetters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'.split(''); aslLetters.forEach(letter => { const desc = aslDescFallback[letter] || 'ASL hand sign'; const el = document.createElement('div'); el.className = 'guide-card'; el.title = `${letter}: ${desc} (click to enlarge)`; el.dataset.letter = letter; el.dataset.hands = '1'; el.dataset.desc = desc; el.dataset.lang = 'ASL'; el.dataset.imgSrc = `asl_signs/${letter}.png`; el.innerHTML = `
ASL Sign ${letter}
${letter} 1 Hand

${desc}

`; el.addEventListener('click', () => openSignLightbox(el)); aslGrid.appendChild(el); }); } function openGuide() { renderGuideCards(); renderAslCards(); if (panelGuide) { try { if (panelGuide.showPopover) { panelGuide.showPopover(); } else { panelGuide.style.display = 'flex'; panelGuide.classList.add('open'); } } catch (err) { panelGuide.style.display = 'flex'; panelGuide.classList.add('open'); } } } function closeGuide() { if (panelGuide) { try { if (panelGuide.hidePopover) { panelGuide.hidePopover(); } else { panelGuide.style.display = 'none'; panelGuide.classList.remove('open'); } } catch (err) { panelGuide.style.display = 'none'; panelGuide.classList.remove('open'); } } } if (panelGuide) { // Synchronize inline display style with popover state panelGuide.addEventListener('toggle', (e) => { if (e.newState === 'open') { panelGuide.style.display = 'flex'; panelGuide.classList.add('open'); } else { panelGuide.style.display = 'none'; panelGuide.classList.remove('open'); } }); // Ensure it starts closed and strictly hidden panelGuide.style.display = 'none'; panelGuide.classList.remove('open'); } const stageGuideBtn = document.getElementById('stage-guide-btn'); if (stageGuideBtn) { stageGuideBtn.addEventListener('click', (e) => { e.preventDefault(); openGuide(); }); } if (btnOpenGuide) { btnOpenGuide.addEventListener('click', (e) => { e.preventDefault(); openGuide(); }); } if (btnCloseGuide) { btnCloseGuide.addEventListener('click', (e) => { e.preventDefault(); closeGuide(); }); } if (tabIsl && tabAsl && contentIsl && contentAsl) { tabIsl.addEventListener('click', () => { tabIsl.classList.add('active'); tabAsl.classList.remove('active'); tabIsl.setAttribute('aria-selected', 'true'); tabAsl.setAttribute('aria-selected', 'false'); contentIsl.classList.remove('hidden'); contentAsl.classList.add('hidden'); }); tabAsl.addEventListener('click', () => { tabAsl.classList.add('active'); tabIsl.classList.remove('active'); tabAsl.setAttribute('aria-selected', 'true'); tabIsl.setAttribute('aria-selected', 'false'); contentAsl.classList.remove('hidden'); contentIsl.classList.add('hidden'); }); } // ── Sign Lightbox popup ───────────────────────────────────────────────── const lightboxOverlay = document.getElementById('sign-lightbox-overlay'); const lightboxImg = document.getElementById('sign-lightbox-img'); const lightboxLetter = document.getElementById('sign-lightbox-letter'); const lightboxType = document.getElementById('sign-lightbox-type'); const lightboxDesc = document.getElementById('sign-lightbox-desc'); const lightboxLang = document.getElementById('sign-lightbox-lang'); const lightboxCloseBtn = document.getElementById('sign-lightbox-close'); function openSignLightbox(dataOrEl) { if (!lightboxOverlay) return; let letter = '?'; let hands = '1'; let desc = ''; let lang = 'ISL'; let imgSrc = ''; if (dataOrEl instanceof HTMLElement) { letter = dataOrEl.dataset.letter || '?'; hands = dataOrEl.dataset.hands || '1'; desc = dataOrEl.dataset.desc || ''; lang = dataOrEl.dataset.lang || 'ISL'; imgSrc = dataOrEl.dataset.imgSrc || ''; } else if (typeof dataOrEl === 'object' && dataOrEl !== null) { letter = dataOrEl.letter || '?'; hands = String(dataOrEl.hands || '1'); desc = dataOrEl.desc || ''; lang = dataOrEl.lang || 'ISL'; imgSrc = dataOrEl.imgSrc || ''; } if (lightboxImg) lightboxImg.src = imgSrc; if (lightboxLetter) lightboxLetter.textContent = `Sign: ${letter}`; if (lightboxType) { const isTwoHand = hands === '2'; lightboxType.textContent = isTwoHand ? '✌️ 2 Hands Required' : '🤚 1 Hand Only'; lightboxType.className = `sign-lightbox-type ${isTwoHand ? 'two-hand' : 'one-hand'}`; } if (lightboxDesc) lightboxDesc.textContent = desc; if (lightboxLang) lightboxLang.textContent = (lang.toUpperCase() === 'ASL') ? '🇺🇸 American Sign Language (ASL)' : '🇮🇳 Indian Sign Language (ISL)'; // Open natively in the browser Top Layer (above all popovers and dialogs) if (typeof lightboxOverlay.showModal === 'function') { if (!lightboxOverlay.open) { lightboxOverlay.showModal(); } } else { lightboxOverlay.setAttribute('open', ''); } } function closeSignLightbox() { if (!lightboxOverlay) return; if (typeof lightboxOverlay.close === 'function') { if (lightboxOverlay.open) { lightboxOverlay.close(); } } else { lightboxOverlay.removeAttribute('open'); } } // Make openSignLightbox available globally for card click handlers window.openSignLightbox = openSignLightbox; // Close button if (lightboxCloseBtn) { lightboxCloseBtn.addEventListener('click', (e) => { e.preventDefault(); e.stopPropagation(); closeSignLightbox(); }); } // Click outside (on backdrop) to close dialog if (lightboxOverlay) { lightboxOverlay.addEventListener('click', (e) => { if (e.target === lightboxOverlay) { closeSignLightbox(); } }); } // ESC key natively closes , but add safety handler document.addEventListener('keydown', (e) => { if (e.key === 'Escape' && lightboxOverlay && lightboxOverlay.open) { closeSignLightbox(); } }); window._renderGuideCards = renderGuideCards; window._openSignGuide = openGuide; } // ── Startup ────────────────────────────────────────────────────────────────── window.addEventListener('DOMContentLoaded', async () => { displayCanvas.width = 640; displayCanvas.height = 480; // Wire controls FIRST: a camera failure must not leave every button dead. setupControls(); initPracticeStudio(); initSignGuide(); initUploadStudio(); setStatus('loading', 'Loading…'); try { // 1. Probe for Flask backend (async, result used by classify()) probeServer(); // 2. Always load JS MLP weights for offline fallback await loadLandmarkModel(); // 3. Start camera await initMediaPipe(); } catch (err) { showCameraError(err); } }); // ══════════════════════════════════════════════════════════════════════════════ // VISUALLY (PRACTICE STUDIO) IMPLEMENTATION // ══════════════════════════════════════════════════════════════════════════════ let practiceTemplates = { asl: {}, isl: {} }; let practiceDialect = 'asl'; // 'asl' | 'isl' let practiceSpeed = 1.0; // 0.5 | 1.0 | 1.5 // Sequence and animation state let practiceSequence = []; // e.g. ['H', 'I', ' ', 'B', 'A', 'B', 'Y'] let practiceCurrentIndex = 0; let practiceIsPlaying = false; let practiceStepStartTime = 0; let practiceAnimFrameId = null; // Poses for interpolation: arrays of 21 landmarks [x, y, z] let practiceFromPose = null; let practiceToPose = null; let practiceCurrentPose = null; // Default neutral resting pose (21 landmarks) const NEUTRAL_REST_POSE = Array(21).fill(0).map((_, i) => { if (i === 0) return [0, 0, 0]; const fingerIdx = Math.floor((i - 1) / 4); const jointIdx = ((i - 1) % 4) + 1; const xOffset = (fingerIdx - 2) * 0.18; const yOffset = -0.35 * jointIdx; return [xOffset, yOffset, 0]; }); function clone21(pts) { if (!pts || !Array.isArray(pts)) return null; return pts.map(pt => [pt[0], pt[1], pt[2] || 0]); } function normalizePose(raw) { if (!raw) { return { twoHanded: false, left: null, right: clone21(NEUTRAL_REST_POSE), wristL: [0.0, 0.0], wristR: [0.0, 0.0], desc: '' }; } if (Array.isArray(raw)) { return { twoHanded: false, left: null, right: clone21(raw), wristL: [0.0, 0.0], wristR: [0.0, 0.0], desc: '' }; } return { twoHanded: !!raw.twoHanded, left: raw.left ? clone21(raw.left) : null, right: raw.right ? clone21(raw.right) : clone21(NEUTRAL_REST_POSE), wristL: raw.wristL ? [raw.wristL[0], raw.wristL[1]] : [0.0, 0.0], wristR: raw.wristR ? [raw.wristR[0], raw.wristR[1]] : [0.0, 0.0], desc: raw.desc || '' }; } function clonePose(pose) { const norm = normalizePose(pose); return { twoHanded: norm.twoHanded, left: clone21(norm.left), right: clone21(norm.right), wristL: [...norm.wristL], wristR: [...norm.wristR], desc: norm.desc }; } function initPracticeStudio() { const canvas = document.getElementById('avatar-canvas'); if (!canvas) return; const ctx = canvas.getContext('2d'); // 1. Fetch templates fetch('./sign_templates.json?v=' + Date.now(), { cache: 'no-store' }) .then(res => { if (!res.ok) throw new Error('HTTP ' + res.status); return res.json(); }) .then(data => { practiceTemplates = data; console.log('[PracticeStudio] Loaded sign templates:', Object.keys(data.asl || {}).length, 'ASL,', Object.keys(data.isl || {}).length, 'ISL'); // Set initial pose practiceCurrentPose = clonePose(getTemplatePose('REST')); window.practiceCurrentPose = practiceCurrentPose; // ── Initialize 3D Avatar (Feature Flag: window.USE_3D_AVATAR) ── initAvatar3DRenderer(); drawAvatar(); const panelGuide = document.getElementById('panel-guide'); if (panelGuide && (panelGuide.matches(':popover-open') || panelGuide.classList.contains('open'))) { if (window._renderGuideCards) window._renderGuideCards(); } }) .catch(err => { console.warn('[PracticeStudio] Failed to load sign_templates.json:', err); }); function initAvatar3DRenderer() { const loadingOverlay = document.getElementById('avatar-3d-loading-overlay'); const loadingStatus = document.getElementById('avatar-loading-status'); const loadingBar = document.getElementById('avatar-loading-bar'); const avatarStatusText = document.getElementById('avatar-status-text'); if (window.USE_3D_AVATAR === false) { if (globalAvatar3D) { globalAvatar3D.dispose(); globalAvatar3D = null; window.globalAvatar3D = null; } window._avatar3DFallbackActive = true; if (loadingOverlay) loadingOverlay.style.display = 'none'; drawAvatar(); return; } window._avatar3DFallbackActive = false; if (loadingOverlay) { loadingOverlay.classList.remove('loaded'); loadingOverlay.style.display = 'flex'; } if (avatarStatusText) avatarStatusText.textContent = 'Loading 3D…'; const canvas3D = document.getElementById('avatar-canvas-3d'); if (!globalAvatar3D && canvas3D) { try { const restHand = practiceTemplates?.asl?.REST || null; globalAvatar3D = new AvatarCanvas(canvas3D, { modelUrl: './assets/Ch46_nonPBR.fbx', restHand: restHand, onProgress: (xhr) => { if (!loadingBar || !loadingStatus) return; if (xhr && xhr.fromCache) { loadingBar.style.width = '100%'; loadingStatus.textContent = 'Instant load from local cache (0 MB downloaded)…'; return; } if (xhr && xhr.lengthComputable && xhr.total > 0) { const pct = Math.min(100, Math.round((xhr.loaded / xhr.total) * 100)); const loadedMB = (xhr.loaded / (1024 * 1024)).toFixed(1); const totalMB = (xhr.total / (1024 * 1024)).toFixed(1); loadingBar.style.width = pct + '%'; loadingStatus.textContent = `Downloading 3D Studio Avatar (${pct}% • ${loadedMB}/${totalMB} MB)…`; } else if (xhr && xhr.loaded > 0) { const loadedMB = (xhr.loaded / (1024 * 1024)).toFixed(1); loadingStatus.textContent = `Downloading 3D Studio Avatar (${loadedMB} MB)…`; loadingBar.style.width = '70%'; } }, onReady: () => { console.log('[PracticeStudio] 3D Avatar active and tracking practice playback.'); if (ctx && canvas) { ctx.clearRect(0, 0, canvas.width, canvas.height); } if (avatarStatusText) avatarStatusText.textContent = 'Ready'; if (loadingBar) loadingBar.style.width = '100%'; if (loadingStatus) loadingStatus.textContent = '3D Avatar Ready!'; if (loadingOverlay) { loadingOverlay.classList.add('loaded'); setTimeout(() => { if (loadingOverlay && loadingOverlay.classList.contains('loaded')) { loadingOverlay.style.display = 'none'; } }, 450); } }, onError: (err) => { console.info('[PracticeStudio] Falling back to 2D character renderer while 3D model is not present:', err); window._avatar3DFallbackActive = true; if (loadingOverlay) loadingOverlay.style.display = 'none'; if (avatarStatusText) avatarStatusText.textContent = 'Ready (2D)'; drawAvatar(); } }); window.globalAvatar3D = globalAvatar3D; } catch (err) { console.warn('[PracticeStudio] Failed to initialize Three.js AvatarCanvas:', err); window._avatar3DFallbackActive = true; if (loadingOverlay) loadingOverlay.style.display = 'none'; drawAvatar(); } } } window._reinitAvatarRenderer = initAvatar3DRenderer; // 2. DOM elements const txtInput = document.getElementById('practice-input'); const charCount = document.getElementById('practice-char-count'); const btnMic = document.getElementById('btn-mic'); const micLabel = document.getElementById('mic-label'); const btnSpeakInput = document.getElementById('btn-speak-input'); const btnClearPractice = document.getElementById('btn-clear-practice'); const btnStartSign = document.getElementById('btn-start-sign'); const sequenceChipsContainer = document.getElementById('sequence-chips'); const seqCounter = document.getElementById('seq-counter'); const btnPracAsl = document.getElementById('btn-practice-asl'); const btnPracIsl = document.getElementById('btn-practice-isl'); const hudChar = document.getElementById('hud-letter-char'); const hudSub = document.getElementById('hud-letter-sub'); const hudCue = document.getElementById('hud-letter-cue'); const hudStatus = document.getElementById('avatar-hud-status'); const avatarStatusText = document.getElementById('avatar-status-text'); // Top-Right Sign Gesture Image Preview Reference const signPreviewWrap = document.getElementById('avatar-sign-preview'); const signPreviewImg = document.getElementById('avatar-sign-img'); const signPreviewBadge = document.getElementById('avatar-sign-badge'); const signPreviewChar = document.getElementById('avatar-sign-char'); const signPreviewHands = document.getElementById('avatar-sign-hands'); const signPreviewDesc = document.getElementById('avatar-sign-desc'); if (signPreviewWrap) { signPreviewWrap.addEventListener('click', () => { if (signPreviewImg && signPreviewImg.dataset.letter) { if (typeof openSignLightbox === 'function') { openSignLightbox({ letter: signPreviewImg.dataset.letter, hands: signPreviewImg.dataset.hands || '1', desc: signPreviewImg.dataset.desc || '', lang: signPreviewImg.dataset.lang || 'ISL', imgSrc: signPreviewImg.src }); } } }); } const btnPrev = document.getElementById('btn-avatar-prev'); const btnPlayPause = document.getElementById('btn-avatar-playpause'); const btnNext = document.getElementById('btn-avatar-next'); const btnReplay = document.getElementById('btn-avatar-replay'); const scrubber = document.getElementById('avatar-scrubber'); // Resize canvas handler function resizeCanvas() { if (!canvas.parentElement) return; const rect = canvas.parentElement.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; const w = Math.floor(rect.width); const h = Math.floor(rect.height); if (w > 0 && h > 0 && (canvas.width !== w * dpr || canvas.height !== h * dpr)) { canvas.width = w * dpr; canvas.height = h * dpr; drawAvatar(); } } window.addEventListener('resize', resizeCanvas); setTimeout(resizeCanvas, 100); // Get template for a character function getTemplatePose(char) { const dict = practiceTemplates[practiceDialect] || practiceTemplates.asl || {}; const upper = String(char).toUpperCase(); if (dict[upper]) return normalizePose(dict[upper]); if (dict['REST']) return normalizePose(dict['REST']); return normalizePose(null); } const ASL_SAMPLES = ['HI', 'HELLO', 'THANK YOU', 'HOW ARE YOU', 'BABY']; const ISL_SAMPLES = ['NAMASTE', 'A B C', '1 2 3', 'ONAM', 'INDIA']; function updateSampleChips(samples) { const chipsContainer = document.querySelector('.quick-samples'); if (!chipsContainer) return; const chips = chipsContainer.querySelectorAll('.sample-chip'); chips.forEach((chip, idx) => { if (samples[idx] !== undefined) { chip.dataset.text = samples[idx]; chip.textContent = samples[idx]; chip.style.display = ''; } else { chip.style.display = 'none'; } }); } // Dialect switch inside Practice if (btnPracAsl && btnPracIsl) { btnPracAsl.addEventListener('click', () => { practiceDialect = 'asl'; btnPracAsl.classList.add('active'); btnPracIsl.classList.remove('active'); updateSampleChips(ASL_SAMPLES); rebuildSequence(false); showToast('Practice dialect: ASL'); }); btnPracIsl.addEventListener('click', () => { practiceDialect = 'isl'; btnPracIsl.classList.add('active'); btnPracAsl.classList.remove('active'); updateSampleChips(ISL_SAMPLES); rebuildSequence(false); showToast('Practice dialect: ISL'); }); } // Speed controls document.querySelectorAll('.speed-btn').forEach(btn => { btn.addEventListener('click', () => { document.querySelectorAll('.speed-btn').forEach(b => b.classList.remove('active')); btn.classList.add('active'); practiceSpeed = parseFloat(btn.dataset.speed) || 1.0; }); }); // Quick sample chips document.querySelectorAll('.sample-chip').forEach(chip => { chip.addEventListener('click', () => { const text = chip.dataset.text || ''; if (txtInput) { txtInput.value = text; updateCharCount(); } rebuildSequence(true); }); }); function updateCharCount() { if (!txtInput || !charCount) return; charCount.textContent = `${txtInput.value.length}/200`; } if (txtInput) { txtInput.addEventListener('input', () => { updateCharCount(); rebuildSequence(false); }); txtInput.addEventListener('keydown', (e) => { if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); rebuildSequence(true); } }); } if (btnClearPractice) { btnClearPractice.addEventListener('click', () => { if (txtInput) txtInput.value = ''; updateCharCount(); practiceSequence = []; practiceCurrentIndex = 0; pausePlayback(); renderSequenceChips(); updateHUD(); practiceCurrentPose = clonePose(getTemplatePose('REST')); drawAvatar(); }); } // Voice Speech-to-Text if (btnMic) { let recognition = null; const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; if (SpeechRecognition) { recognition = new SpeechRecognition(); recognition.continuous = false; recognition.interimResults = false; recognition.lang = 'en-US'; recognition.onstart = () => { btnMic.classList.add('listening'); if (micLabel) micLabel.textContent = 'Listening…'; showToast('Speak now — listening…'); }; recognition.onresult = (event) => { const transcript = event.results[0][0].transcript; if (txtInput) { txtInput.value = transcript.toUpperCase(); updateCharCount(); rebuildSequence(true); } }; recognition.onerror = (event) => { console.warn('[PracticeStudio] Speech recognition error:', event.error); btnMic.classList.remove('listening'); if (micLabel) micLabel.textContent = 'Speak'; showToast('Voice error: ' + event.error); }; recognition.onend = () => { btnMic.classList.remove('listening'); if (micLabel) micLabel.textContent = 'Speak'; }; btnMic.addEventListener('click', () => { try { if (btnMic.classList.contains('listening')) { recognition.stop(); } else { recognition.start(); } } catch (err) { console.warn('[PracticeStudio] Speech start exception:', err); } }); } else { btnMic.addEventListener('click', () => { showToast('Speech recognition not supported in this browser. Please type.'); }); } } // TTS speak aloud if (btnSpeakInput) { btnSpeakInput.addEventListener('click', () => { const text = (txtInput ? txtInput.value.trim() : ''); if (!text) { showToast('Type some words to speak aloud.'); return; } if ('speechSynthesis' in window) { window.speechSynthesis.cancel(); const utter = new SpeechSynthesisUtterance(text); utter.rate = 0.95; window.speechSynthesis.speak(utter); } }); } // Start Demonstration Button if (btnStartSign) { btnStartSign.addEventListener('click', () => { rebuildSequence(true); }); } // Parse text into sequence of signs function rebuildSequence(autoPlay = false) { const raw = (txtInput ? txtInput.value : '').toUpperCase(); const cleanChars = []; for (let i = 0; i < raw.length; i++) { const ch = raw[i]; if ((ch >= 'A' && ch <= 'Z') || (ch >= '0' && ch <= '9')) { cleanChars.push(ch); } else if (ch === ' ' && cleanChars.length > 0 && cleanChars[cleanChars.length - 1] !== ' ') { cleanChars.push(' '); } } // Remove trailing space if (cleanChars.length > 0 && cleanChars[cleanChars.length - 1] === ' ') { cleanChars.pop(); } practiceSequence = cleanChars; practiceCurrentIndex = 0; renderSequenceChips(); updateHUD(); if (practiceSequence.length === 0) { practiceCurrentPose = clonePose(getTemplatePose('REST')); drawAvatar(); return; } if (autoPlay) { startPlayback(); } else { // While typing, update pose & chips without any aggressive scroll triggering practiceCurrentIndex = 0; const char = practiceSequence[0] || 'REST'; practiceToPose = clonePose(getTemplatePose(char === ' ' ? 'REST' : char)); practiceFromPose = practiceCurrentPose ? clonePose(practiceCurrentPose) : clonePose(getTemplatePose('REST')); practiceStepStartTime = performance.now(); renderSequenceChips(); updateHUD(); if (sequenceChipsContainer) { sequenceChipsContainer.scrollLeft = 0; } } } function renderSequenceChips() { if (!sequenceChipsContainer) return; sequenceChipsContainer.innerHTML = ''; if (practiceSequence.length === 0) { sequenceChipsContainer.innerHTML = 'Enter text above to preview gesture timeline…'; if (seqCounter) seqCounter.textContent = '0 / 0'; if (scrubber) scrubber.value = 0; return; } if (seqCounter) { seqCounter.textContent = `${practiceCurrentIndex + 1} / ${practiceSequence.length}`; } practiceSequence.forEach((char, idx) => { const chip = document.createElement('button'); chip.className = 'sequence-chip' + (char === ' ' ? ' is-space' : '') + (idx === practiceCurrentIndex ? ' active' : ''); chip.textContent = (char === ' ' ? '␣' : char); chip.title = (char === ' ' ? 'Space (Rest pause)' : `Letter ${char}`); chip.addEventListener('click', () => { jumpToStep(idx); }); sequenceChipsContainer.appendChild(chip); }); if (scrubber) { const pct = (practiceSequence.length > 1) ? (practiceCurrentIndex / (practiceSequence.length - 1)) * 100 : 0; scrubber.value = pct; } } function updateHUD() { if (practiceSequence.length === 0) { if (hudChar) hudChar.textContent = '—'; if (hudSub) hudSub.textContent = 'Waiting for input'; if (hudCue) { hudCue.style.display = 'none'; hudCue.textContent = ''; } if (avatarStatusText) avatarStatusText.textContent = 'Ready'; if (hudStatus) hudStatus.classList.remove('playing'); if (signPreviewWrap) signPreviewWrap.style.display = 'none'; return; } const currChar = practiceSequence[practiceCurrentIndex]; if (hudChar) { hudChar.textContent = (currChar === ' ' ? '␣' : currChar); } const currPose = getTemplatePose(currChar === ' ' ? 'REST' : currChar); const is2H = currPose.twoHanded; const desc = currPose.desc; if (hudSub) { if (currChar === ' ') { hudSub.textContent = `Word Boundary Pause (${practiceCurrentIndex + 1}/${practiceSequence.length})`; } else { const handTag = (practiceDialect === 'isl') ? (is2H ? ' • Dual Hand (Two-Handed)' : ' • Single Hand') : ' • Single Hand'; hudSub.textContent = `${practiceDialect.toUpperCase()} Sign • Step ${practiceCurrentIndex + 1} of ${practiceSequence.length}${handTag}`; } } if (hudCue) { if (desc && currChar !== ' ') { hudCue.style.display = 'inline-flex'; hudCue.textContent = `${is2H ? '👐 Two-Handed: ' : '✋ '}${desc}`; } else { hudCue.style.display = 'none'; hudCue.textContent = ''; } } if (avatarStatusText) { avatarStatusText.textContent = practiceIsPlaying ? 'Demonstrating…' : 'Paused'; } if (hudStatus) { if (practiceIsPlaying) hudStatus.classList.add('playing'); else hudStatus.classList.remove('playing'); } // Update Top-Right Sign Gesture Image Preview Reference if (signPreviewWrap) { if (currChar && currChar !== ' ') { const lang = (practiceDialect || 'isl').toLowerCase(); const imgSrc = (lang === 'asl') ? `asl_signs/${currChar}.png` : `isl_signs/${currChar}.png`; if (signPreviewImg) { signPreviewImg.src = imgSrc; signPreviewImg.dataset.letter = currChar; signPreviewImg.dataset.hands = is2H ? '2' : '1'; signPreviewImg.dataset.desc = desc || ''; signPreviewImg.dataset.lang = lang.toUpperCase(); } if (signPreviewBadge) signPreviewBadge.textContent = lang.toUpperCase(); if (signPreviewChar) signPreviewChar.textContent = `Sign: ${currChar}`; if (signPreviewHands) signPreviewHands.textContent = is2H ? '2 Hands' : '1 Hand'; if (signPreviewDesc) signPreviewDesc.textContent = desc || (is2H ? 'Dual-hand gesture' : 'Single-hand gesture'); signPreviewWrap.style.display = 'flex'; } else { signPreviewWrap.style.display = 'none'; } } } function jumpToStep(idx) { if (practiceSequence.length === 0) return; practiceCurrentIndex = Math.max(0, Math.min(idx, practiceSequence.length - 1)); const char = practiceSequence[practiceCurrentIndex]; practiceFromPose = practiceCurrentPose ? clonePose(practiceCurrentPose) : clonePose(getTemplatePose('REST')); practiceToPose = clonePose(getTemplatePose(char === ' ' ? 'REST' : char)); practiceStepStartTime = performance.now(); renderSequenceChips(); updateHUD(); // Scroll active chip into view horizontally inside sequenceChipsContainer ONLY // NEVER use activeChip.scrollIntoView() because it forces the entire window/page to scroll down! const activeChip = sequenceChipsContainer ? sequenceChipsContainer.children[practiceCurrentIndex] : null; if (activeChip && sequenceChipsContainer && typeof sequenceChipsContainer.scrollTo === 'function') { const targetScroll = activeChip.offsetLeft - (sequenceChipsContainer.clientWidth - activeChip.clientWidth) / 2; sequenceChipsContainer.scrollTo({ left: Math.max(0, targetScroll), behavior: 'smooth' }); } } function startPlayback() { if (practiceSequence.length === 0) return; practiceIsPlaying = true; if (btnPlayPause) btnPlayPause.textContent = '⏸'; jumpToStep(practiceCurrentIndex); updateHUD(); } function pausePlayback() { practiceIsPlaying = false; if (btnPlayPause) btnPlayPause.textContent = '▶'; updateHUD(); } // Playback control buttons if (btnPlayPause) { btnPlayPause.addEventListener('click', () => { if (practiceIsPlaying) { pausePlayback(); } else { if (practiceCurrentIndex >= practiceSequence.length - 1) { practiceCurrentIndex = 0; } startPlayback(); } }); } if (btnPrev) { btnPrev.addEventListener('click', () => { pausePlayback(); jumpToStep(practiceCurrentIndex - 1); }); } if (btnNext) { btnNext.addEventListener('click', () => { pausePlayback(); jumpToStep(practiceCurrentIndex + 1); }); } if (btnReplay) { btnReplay.addEventListener('click', () => { practiceCurrentIndex = 0; startPlayback(); }); } if (scrubber) { scrubber.addEventListener('input', () => { if (practiceSequence.length === 0) return; pausePlayback(); const val = parseFloat(scrubber.value); const targetIdx = Math.round((val / 100) * (practiceSequence.length - 1)); jumpToStep(targetIdx); }); } // Interpolation & Render Loop function lerp(a, b, t) { return a + (b - a) * t; } function easeInOutCubic(x) { return x < 0.5 ? 4 * x * x * x : 1 - Math.pow(-2 * x + 2, 3) / 2; } function avatarAnimationLoop(now) { practiceAnimFrameId = requestAnimationFrame(avatarAnimationLoop); // Timing for step transitions (smoother spring-like curve at high refresh rate) const transitionMs = 600 / practiceSpeed; const holdMs = 850 / practiceSpeed; const totalStepMs = transitionMs + holdMs; // Organic micro-breathing & idle motion for natural 60 FPS live feel const breathT = (now * 0.0022); const breathOffset = Math.sin(breathT) * 1.5; if (practiceToPose && practiceFromPose) { const elapsed = now - practiceStepStartTime; const rawProgress = Math.min(1.0, elapsed / transitionMs); // Quintic smoothstep for butter-smooth 60fps acceleration and deceleration const t = rawProgress < 0.5 ? 16 * Math.pow(rawProgress, 5) : 1 - Math.pow(-2 * rawProgress + 2, 5) / 2; if (!practiceCurrentPose) { practiceCurrentPose = clonePose(practiceFromPose); } // Interpolate right hand const fromR = practiceFromPose.right || NEUTRAL_REST_POSE; const toR = practiceToPose.right || NEUTRAL_REST_POSE; if (!practiceCurrentPose.right) practiceCurrentPose.right = clone21(fromR); for (let i = 0; i < 21; i++) { practiceCurrentPose.right[i][0] = lerp(fromR[i][0], toR[i][0], t); practiceCurrentPose.right[i][1] = lerp(fromR[i][1], toR[i][1], t); practiceCurrentPose.right[i][2] = lerp(fromR[i][2], toR[i][2], t); } // Interpolate left hand const fromL = practiceFromPose.left; const toL = practiceToPose.left; if (fromL || toL) { const effFromL = fromL || NEUTRAL_REST_POSE; const effToL = toL || NEUTRAL_REST_POSE; if (!practiceCurrentPose.left) practiceCurrentPose.left = clone21(effFromL); for (let i = 0; i < 21; i++) { practiceCurrentPose.left[i][0] = lerp(effFromL[i][0], effToL[i][0], t); practiceCurrentPose.left[i][1] = lerp(effFromL[i][1], effToL[i][1], t); practiceCurrentPose.left[i][2] = lerp(effFromL[i][2], effToL[i][2], t); } practiceCurrentPose.twoHanded = (t > 0.5) ? practiceToPose.twoHanded : practiceFromPose.twoHanded; } else { practiceCurrentPose.left = null; practiceCurrentPose.twoHanded = false; } // Interpolate wrist offsets with subtle micro-sway practiceCurrentPose.wristL = [ lerp(practiceFromPose.wristL ? practiceFromPose.wristL[0] : 0, practiceToPose.wristL ? practiceToPose.wristL[0] : 0, t), lerp(practiceFromPose.wristL ? practiceFromPose.wristL[1] : 0, practiceToPose.wristL ? practiceToPose.wristL[1] : 0, t) ]; practiceCurrentPose.wristR = [ lerp(practiceFromPose.wristR ? practiceFromPose.wristR[0] : 0, practiceToPose.wristR ? practiceToPose.wristR[0] : 0, t), lerp(practiceFromPose.wristR ? practiceFromPose.wristR[1] : 0, practiceToPose.wristR ? practiceToPose.wristR[1] : 0, t) ]; practiceCurrentPose.desc = practiceToPose.desc; practiceCurrentPose.breathOffset = breathOffset; window.practiceCurrentPose = practiceCurrentPose; if (practiceIsPlaying && elapsed >= totalStepMs) { if (practiceCurrentIndex < practiceSequence.length - 1) { jumpToStep(practiceCurrentIndex + 1); } else { // Completed full sequence pausePlayback(); if (avatarStatusText) avatarStatusText.textContent = 'Complete'; } } } // When 3D Avatar is enabled and active, skip 2D canvas drawing completely if (window.USE_3D_AVATAR !== false && !window._avatar3DFallbackActive) return; drawAvatar(); } // Draw 21-landmark hand skeleton with 5 distinct finger colors, palm metacarpals, and pearl beads function drawHandSkeleton(ctx, pose21, wristX, wristY, dpr, scale, isLeft) { if (!pose21 || pose21.length !== 21) return; // Project landmarks to screen coordinates const pts = pose21.map(pt => { const lx = pt[0]; const ly = pt[1]; const lz = pt[2] || 0; const depth = 1.0 / (1.0 - lz * 0.18); return [ wristX + lx * scale * depth, wristY + ly * scale * depth, lz ]; }); // Palm Metacarpal Structure (cool titanium slate connecting wrist to the 4 knuckles) const palmCol = 'rgba(145, 160, 175, 0.85)'; ctx.strokeStyle = palmCol; ctx.lineWidth = 3.5 * dpr; // Metacarpal rays: wrist to MCPs (5, 9, 13, 17) [5, 9, 13, 17].forEach(mcp => { ctx.beginPath(); ctx.moveTo(wristX, wristY); ctx.lineTo(pts[mcp][0], pts[mcp][1]); ctx.stroke(); }); // Knuckle transverse arch: connecting MCP 5 -> 9 -> 13 -> 17 ctx.lineWidth = 4.5 * dpr; ctx.beginPath(); ctx.moveTo(pts[5][0], pts[5][1]); ctx.lineTo(pts[9][0], pts[9][1]); ctx.lineTo(pts[13][0], pts[13][1]); ctx.lineTo(pts[17][0], pts[17][1]); ctx.stroke(); // Finger Segments with exact reference colors: // Thumb: Coral / Red-Pink (wrist 0 -> 1 -> 2 -> 3 -> 4) // Index: Deep Electric Blue (5 -> 6 -> 7 -> 8) // Middle: Bright Emerald Green (9 -> 10 -> 11 -> 12) // Ring: Vibrant Cyan (13 -> 14 -> 15 -> 16) // Pinky: Warm Orange (17 -> 18 -> 19 -> 20) const fingerGroups = [ { indices: [0, 1, 2, 3, 4], color: '#ff4d4d', width: 6.0 * dpr }, // Thumb { indices: [5, 6, 7, 8], color: '#2979ff', width: 6.0 * dpr }, // Index { indices: [9, 10, 11, 12], color: '#00e676', width: 6.0 * dpr }, // Middle { indices: [13, 14, 15, 16], color: '#00e5ff', width: 6.0 * dpr }, // Ring { indices: [17, 18, 19, 20], color: '#ff9800', width: 6.0 * dpr }, // Pinky ]; fingerGroups.forEach(group => { ctx.strokeStyle = group.color; ctx.lineWidth = group.width; for (let k = 0; k < group.indices.length - 1; k++) { const i1 = group.indices[k]; const i2 = group.indices[k + 1]; ctx.beginPath(); ctx.moveTo(pts[i1][0], pts[i1][1]); ctx.lineTo(pts[i2][0], pts[i2][1]); ctx.stroke(); } }); // Joint caps: Clean pearl white dots ctx.fillStyle = '#ffffff'; pts.forEach((pt, idx) => { if (idx === 0) return; // Skip wrist, drawn specially below const radius = (idx % 4 === 0) ? 2.5 * dpr : 2.0 * dpr; ctx.beginPath(); ctx.arc(pt[0], pt[1], radius, 0, Math.PI * 2); ctx.fill(); }); // Wrist Joint: Clean pearl white bead ctx.fillStyle = '#ffffff'; ctx.beginPath(); ctx.arc(wristX, wristY, 5 * dpr, 0, Math.PI * 2); ctx.fill(); } // Draw Avatar Canvas function drawAvatar() { if (!canvas || !ctx) return; const dpr = window.devicePixelRatio || 1; const w = canvas.width; const h = canvas.height; // Strict Guard: When 3D mode is active, keep 2D canvas strictly cleared so the old model never flashes if (window.USE_3D_AVATAR !== false && !window._avatar3DFallbackActive) { ctx.clearRect(0, 0, w, h); return; } ctx.save(); ctx.clearRect(0, 0, w, h); // 1. Studio background (theme-aware soft vignette) const isLight = document.documentElement.getAttribute('data-theme') === 'light'; const bgGrad = ctx.createRadialGradient(w * 0.5, h * 0.45, 10, w * 0.5, h * 0.5, Math.max(w, h) * 0.75); if (isLight) { bgGrad.addColorStop(0, '#f8fafc'); bgGrad.addColorStop(0.6, '#edf2f7'); bgGrad.addColorStop(1, '#e2e8f0'); } else { bgGrad.addColorStop(0, '#1c1f26'); bgGrad.addColorStop(0.6, '#14161b'); bgGrad.addColorStop(1, '#0c0d10'); } ctx.fillStyle = bgGrad; ctx.fillRect(0, 0, w, h); // 2. Continuous 60fps Micro-Motion Dynamics const breath = (practiceCurrentPose && practiceCurrentPose.breathOffset !== undefined) ? practiceCurrentPose.breathOffset : 0; // Harmonious Skin & Clothing Palette const skinBase = isLight ? '#f2caa7' : '#d8af8c'; const skinShadow = isLight ? '#dbaa83' : '#be8f6c'; const skinHighlight = isLight ? '#fce5d4' : '#eaceb9'; const hairBase = isLight ? '#2a1f1b' : '#1b1411'; const hairHighlight = isLight ? '#44352e' : '#332722'; const clothesTone = isLight ? '#1e293b' : '#18202f'; const clothesHighlight = isLight ? '#334155' : '#273447'; const collarTone = isLight ? '#e2e8f0' : '#334155'; // Perfectly Balanced Proportions // Head size & center (natural human ratio ~ 1:7.5) const headCx = w * 0.50; const headCy = h * 0.22 + breath * 0.4; const headRx = w * 0.092; const headRy = h * 0.118; // Shoulders & Torso anchors (broad, natural masculine/athletic posture) const shoulderY = h * 0.41 + breath; const leftShoulderX = w * 0.33; const rightShoulderX = w * 0.67; // Neck dimensions const neckTopY = headCy + headRy * 0.70; const neckBottomY = shoulderY + h * 0.02; const neckHalfW = headRx * 0.44; // ── NECK & TRAPEZIUS (Base layer drawn behind head) ── ctx.save(); // Trapezius muscles connecting neck to shoulders const trapGrad = ctx.createLinearGradient(0, headCy + headRy, 0, shoulderY); trapGrad.addColorStop(0, skinShadow); trapGrad.addColorStop(1, skinBase); ctx.fillStyle = trapGrad; ctx.beginPath(); ctx.moveTo(headCx - neckHalfW * 1.05, neckTopY); ctx.quadraticCurveTo(headCx - neckHalfW * 1.5, shoulderY - h * 0.02, leftShoulderX, shoulderY); ctx.lineTo(rightShoulderX, shoulderY); ctx.quadraticCurveTo(headCx + neckHalfW * 1.5, shoulderY - h * 0.02, headCx + neckHalfW * 1.05, neckTopY); ctx.closePath(); ctx.fill(); // Neck cylinder with cylindrical 3D radial/linear shading const neckGrad = ctx.createLinearGradient(headCx - neckHalfW, 0, headCx + neckHalfW, 0); neckGrad.addColorStop(0, skinShadow); neckGrad.addColorStop(0.25, skinBase); neckGrad.addColorStop(0.70, skinHighlight); neckGrad.addColorStop(1, skinShadow); ctx.fillStyle = neckGrad; ctx.beginPath(); ctx.moveTo(headCx - neckHalfW, neckTopY); ctx.lineTo(headCx - neckHalfW * 1.25, neckBottomY); ctx.quadraticCurveTo(headCx, neckBottomY + 8 * dpr, headCx + neckHalfW * 1.25, neckBottomY); ctx.lineTo(headCx + neckHalfW, neckTopY); ctx.closePath(); ctx.fill(); // Deep under-jaw shadow const chinShadowGrad = ctx.createRadialGradient(headCx, neckTopY, 0, headCx, neckTopY, neckHalfW * 1.8); chinShadowGrad.addColorStop(0, 'rgba(0, 0, 0, 0.28)'); chinShadowGrad.addColorStop(0.7, 'rgba(0, 0, 0, 0.12)'); chinShadowGrad.addColorStop(1, 'rgba(0, 0, 0, 0.0)'); ctx.fillStyle = chinShadowGrad; ctx.beginPath(); ctx.ellipse(headCx, neckTopY + 2, neckHalfW * 1.3, 14 * dpr, 0, 0, Math.PI); ctx.fill(); ctx.restore(); // ── TORSO & CLOTHING (Modern crewneck sweater / tee) ── ctx.save(); const torsoGrad = ctx.createLinearGradient(0, shoulderY - 10, 0, h); torsoGrad.addColorStop(0, clothesHighlight); torsoGrad.addColorStop(0.3, clothesTone); torsoGrad.addColorStop(1, isLight ? '#0f172a' : '#0b0f19'); ctx.fillStyle = torsoGrad; ctx.beginPath(); // Neckline collar anchor ctx.moveTo(headCx - neckHalfW * 1.25, shoulderY - h * 0.01); // Smooth left shoulder arch ctx.bezierCurveTo( w * 0.38, shoulderY - h * 0.01, leftShoulderX + w * 0.02, shoulderY - h * 0.005, leftShoulderX, shoulderY + h * 0.015 ); // Left torso side body ctx.lineTo(w * 0.32, h * 0.99); ctx.lineTo(w * 0.68, h * 0.99); // Right torso side body ctx.lineTo(rightShoulderX, shoulderY + h * 0.015); // Smooth right shoulder arch ctx.bezierCurveTo( rightShoulderX - w * 0.02, shoulderY - h * 0.005, w * 0.62, shoulderY - h * 0.01, headCx + neckHalfW * 1.25, shoulderY - h * 0.01 ); ctx.closePath(); ctx.fill(); // Ribbed Crewneck Collar (outer & inner) ctx.fillStyle = collarTone; ctx.beginPath(); ctx.ellipse(headCx, shoulderY + 4 * dpr, neckHalfW * 1.35, 14 * dpr, 0, 0, Math.PI * 2); ctx.fill(); // Inner collar scoop ctx.fillStyle = skinShadow; ctx.beginPath(); ctx.ellipse(headCx, shoulderY + 2 * dpr, neckHalfW * 1.05, 9 * dpr, 0, 0, Math.PI * 2); ctx.fill(); // Clavicle & chest contour creases ctx.strokeStyle = 'rgba(255, 255, 255, 0.07)'; ctx.lineWidth = 2 * dpr; ctx.beginPath(); ctx.moveTo(headCx - neckHalfW * 1.3, shoulderY + 18 * dpr); ctx.quadraticCurveTo(headCx, shoulderY + 28 * dpr, headCx + neckHalfW * 1.3, shoulderY + 18 * dpr); ctx.stroke(); ctx.restore(); // ── EARS (Behind face, natural organic pinna) ── const earY = headCy + headRy * 0.05; const earRx = headRx * 0.20; const earRy = headRy * 0.30; [-1, 1].forEach(dir => { const ex = headCx + dir * (headRx * 0.98); ctx.fillStyle = skinShadow; ctx.beginPath(); ctx.ellipse(ex, earY, earRx, earRy, dir * 0.08, 0, Math.PI * 2); ctx.fill(); // Inner ear fossa fold ctx.fillStyle = 'rgba(0, 0, 0, 0.12)'; ctx.beginPath(); ctx.ellipse(ex - dir * 2, earY, earRx * 0.55, earRy * 0.55, dir * 0.08, 0, Math.PI * 2); ctx.fill(); }); // ── HEAD & JAW SHAPE (Smooth handsome proportioned jawline) ── ctx.save(); const faceGrad = ctx.createRadialGradient( headCx - headRx * 0.2, headCy - headRy * 0.25, headRx * 0.1, headCx, headCy, headRy * 1.15 ); faceGrad.addColorStop(0, skinHighlight); faceGrad.addColorStop(0.55, skinBase); faceGrad.addColorStop(1, skinShadow); ctx.fillStyle = faceGrad; ctx.beginPath(); // Start at temples ctx.moveTo(headCx - headRx, headCy - headRy * 0.2); // Smooth cheek and tapered jaw to chin ctx.bezierCurveTo( headCx - headRx * 0.98, headCy + headRy * 0.40, headCx - headRx * 0.55, headCy + headRy * 0.90, headCx - headRx * 0.24, headCy + headRy ); // Flat rounded chin ctx.quadraticCurveTo(headCx, headCy + headRy * 1.02, headCx + headRx * 0.24, headCy + headRy); // Right cheek and tapered jaw ctx.bezierCurveTo( headCx + headRx * 0.55, headCy + headRy * 0.90, headCx + headRx * 0.98, headCy + headRy * 0.40, headCx + headRx, headCy - headRy * 0.2 ); // Cranium top ctx.bezierCurveTo( headCx + headRx, headCy - headRy * 1.05, headCx - headRx, headCy - headRy * 1.05, headCx - headRx, headCy - headRy * 0.2 ); ctx.closePath(); ctx.fill(); // Subtle natural jawline definition ctx.strokeStyle = isLight ? 'rgba(120, 80, 60, 0.20)' : 'rgba(255, 255, 255, 0.10)'; ctx.lineWidth = 1.6 * dpr; ctx.stroke(); // ── STYLISH MODERN HAIR (Volumetric layered brush-back crop) ── const hairGrad = ctx.createLinearGradient(0, headCy - headRy * 1.25, 0, headCy); hairGrad.addColorStop(0, hairHighlight); hairGrad.addColorStop(1, hairBase); ctx.fillStyle = hairGrad; ctx.beginPath(); ctx.moveTo(headCx - headRx * 1.04, headCy - headRy * 0.15); // Sideburn left ctx.lineTo(headCx - headRx * 0.96, headCy + headRy * 0.08); ctx.lineTo(headCx - headRx * 0.88, headCy - headRy * 0.18); // Volumetric top with texture ctx.bezierCurveTo( headCx - headRx * 1.10, headCy - headRy * 1.05, headCx - headRx * 0.40, headCy - headRy * 1.35, headCx, headCy - headRy * 1.25 ); ctx.bezierCurveTo( headCx + headRx * 0.45, headCy - headRy * 1.35, headCx + headRx * 1.12, headCy - headRy * 1.02, headCx + headRx * 1.04, headCy - headRy * 0.15 ); // Sideburn right ctx.lineTo(headCx + headRx * 0.88, headCy - headRy * 0.18); ctx.lineTo(headCx + headRx * 0.96, headCy + headRy * 0.08); // Hairline across forehead with natural textured sweep ctx.quadraticCurveTo(headCx + headRx * 0.80, headCy - headRy * 0.50, headCx + headRx * 0.50, headCy - headRy * 0.55); ctx.quadraticCurveTo(headCx + headRx * 0.20, headCy - headRy * 0.65, headCx - headRx * 0.10, headCy - headRy * 0.50); ctx.quadraticCurveTo(headCx - headRx * 0.50, headCy - headRy * 0.60, headCx - headRx * 0.90, headCy - headRy * 0.40); ctx.closePath(); ctx.fill(); // Subtle hairline highlight sheen ctx.strokeStyle = hairHighlight; ctx.lineWidth = 2.0 * dpr; ctx.beginPath(); ctx.moveTo(headCx - headRx * 0.6, headCy - headRy * 1.05); ctx.quadraticCurveTo(headCx, headCy - headRy * 1.18, headCx + headRx * 0.5, headCy - headRy * 1.02); ctx.stroke(); // ── EYEBROWS (Natural feathered arches) ── ctx.strokeStyle = hairBase; ctx.lineWidth = 3.6 * dpr; ctx.lineCap = 'round'; const browY = headCy - headRy * 0.15; // Left eyebrow ctx.beginPath(); ctx.moveTo(headCx - headRx * 0.60, browY + 1 * dpr); ctx.quadraticCurveTo(headCx - headRx * 0.38, browY - 6 * dpr, headCx - headRx * 0.14, browY); ctx.stroke(); // Right eyebrow ctx.beginPath(); ctx.moveTo(headCx + headRx * 0.14, browY); ctx.quadraticCurveTo(headCx + headRx * 0.38, browY - 6 * dpr, headCx + headRx * 0.60, browY + 1 * dpr); ctx.stroke(); // ── EYES (Realistic almond contour, iris, pupil, gleam) ── const eyeY = headCy - headRy * 0.01; const eyeSpacing = headRx * 0.38; const eyeW = headRx * 0.22; const eyeH = headRy * 0.11; [-1, 1].forEach(dir => { const ex = headCx + dir * eyeSpacing; // Almond eye sclera ctx.fillStyle = '#ffffff'; ctx.beginPath(); ctx.moveTo(ex - eyeW, eyeY); ctx.quadraticCurveTo(ex, eyeY - eyeH * 1.1, ex + eyeW, eyeY); ctx.quadraticCurveTo(ex, eyeY + eyeH * 0.95, ex - eyeW, eyeY); ctx.closePath(); ctx.fill(); // Soft eyelid outline ctx.strokeStyle = isLight ? 'rgba(70, 40, 30, 0.45)' : 'rgba(255, 230, 220, 0.35)'; ctx.lineWidth = 1.4 * dpr; ctx.stroke(); // Iris (Warm Hazel-Amber) ctx.save(); ctx.beginPath(); ctx.arc(ex, eyeY, eyeH * 0.88, 0, Math.PI * 2); ctx.fillStyle = '#4c3525'; ctx.fill(); // Pupil ctx.fillStyle = '#121215'; ctx.beginPath(); ctx.arc(ex, eyeY, eyeH * 0.48, 0, Math.PI * 2); ctx.fill(); // Catchlight sparkle (vital for life-like presence) ctx.fillStyle = 'rgba(255, 255, 255, 0.9)'; ctx.beginPath(); ctx.arc(ex - eyeW * 0.18, eyeY - eyeH * 0.22, 1.8 * dpr, 0, Math.PI * 2); ctx.fill(); ctx.restore(); // Upper eyelid crease (fold) ctx.strokeStyle = isLight ? 'rgba(100, 60, 45, 0.35)' : 'rgba(255, 255, 255, 0.15)'; ctx.lineWidth = 1.6 * dpr; ctx.beginPath(); ctx.arc(ex, eyeY - 2 * dpr, eyeW * 0.95, Math.PI * 1.15, Math.PI * 1.85, false); ctx.stroke(); }); // ── NOSE (Subtle sculpted bridge and philtrum) ── const noseY = headCy + headRy * 0.26; ctx.strokeStyle = isLight ? 'rgba(130, 85, 65, 0.42)' : 'rgba(255, 210, 190, 0.35)'; ctx.lineWidth = 2.0 * dpr; ctx.lineCap = 'round'; ctx.beginPath(); ctx.moveTo(headCx - 1 * dpr, headCy + headRy * 0.06); ctx.lineTo(headCx - 2 * dpr, noseY); ctx.quadraticCurveTo(headCx, noseY + 3 * dpr, headCx + 4 * dpr, noseY); ctx.stroke(); // ── MOUTH (Warm, confident, natural masculine lips) ── const mouthY = headCy + headRy * 0.52; const mouthW = headRx * 0.32; // Upper lip ctx.fillStyle = isLight ? '#c86f66' : '#ad5950'; ctx.beginPath(); ctx.moveTo(headCx - mouthW, mouthY); ctx.quadraticCurveTo(headCx - mouthW * 0.3, mouthY - 3 * dpr, headCx, mouthY - 1 * dpr); ctx.quadraticCurveTo(headCx + mouthW * 0.3, mouthY - 3 * dpr, headCx + mouthW, mouthY); ctx.quadraticCurveTo(headCx, mouthY + 3 * dpr, headCx - mouthW, mouthY); ctx.fill(); // Lower lip ctx.fillStyle = isLight ? '#d47b74' : '#ba665f'; ctx.beginPath(); ctx.moveTo(headCx - mouthW * 0.75, mouthY + 1 * dpr); ctx.quadraticCurveTo(headCx, mouthY + headRy * 0.12, headCx + mouthW * 0.75, mouthY + 1 * dpr); ctx.quadraticCurveTo(headCx, mouthY + 2 * dpr, headCx - mouthW * 0.75, mouthY + 1 * dpr); ctx.fill(); // Lip center line ctx.strokeStyle = isLight ? '#993d36' : '#7a2d27'; ctx.lineWidth = 1.4 * dpr; ctx.beginPath(); ctx.moveTo(headCx - mouthW, mouthY); ctx.quadraticCurveTo(headCx, mouthY + 2 * dpr, headCx + mouthW, mouthY); ctx.stroke(); ctx.restore(); // restore face canvas state // ── VOLUMETRIC SLEEVES & ANATOMICAL ARMS ── // Seamless limb renderer with smooth joint caps and shading function drawHumanArm(shoulderX, shoulderY, elbowX, elbowY, wristX, wristY) { const sleeveWidth = 18 * dpr; const bicepWidth = 14 * dpr; const elbowWidth = 12 * dpr; const wristWidth = 9.5 * dpr; const sleeveGrad = ctx.createLinearGradient(shoulderX, shoulderY, elbowX, elbowY); sleeveGrad.addColorStop(0, clothesHighlight); sleeveGrad.addColorStop(1, clothesTone); const skinGrad = ctx.createLinearGradient(elbowX, elbowY, wristX, wristY); skinGrad.addColorStop(0, skinShadow); skinGrad.addColorStop(0.5, skinBase); skinGrad.addColorStop(1, skinHighlight); const strokeCol = isLight ? 'rgba(40, 25, 20, 0.18)' : 'rgba(255, 255, 255, 0.10)'; // 1. Upper Arm (Short Sleeve Shirt) const dx1 = elbowX - shoulderX; const dy1 = elbowY - shoulderY; const len1 = Math.sqrt(dx1 * dx1 + dy1 * dy1) || 1; const nx1 = -dy1 / len1; const ny1 = dx1 / len1; // Draw shoulder ball cap for seamless blending into torso ctx.fillStyle = sleeveGrad; ctx.beginPath(); ctx.arc(shoulderX, shoulderY, sleeveWidth, 0, Math.PI * 2); ctx.fill(); // Upper arm sleeve body ctx.beginPath(); ctx.moveTo(shoulderX + nx1 * sleeveWidth, shoulderY + ny1 * sleeveWidth); ctx.lineTo(elbowX + nx1 * bicepWidth, elbowY + ny1 * bicepWidth); ctx.arc(elbowX, elbowY, bicepWidth, Math.atan2(ny1, nx1), Math.atan2(-ny1, -nx1), false); ctx.lineTo(shoulderX - nx1 * sleeveWidth, shoulderY - ny1 * sleeveWidth); ctx.closePath(); ctx.fill(); // Sleeve hem rim ctx.strokeStyle = collarTone; ctx.lineWidth = 2.0 * dpr; ctx.beginPath(); ctx.arc(elbowX, elbowY, bicepWidth, Math.atan2(ny1, nx1), Math.atan2(-ny1, -nx1), false); ctx.stroke(); // 2. Forearm (Smooth Tapered Skin from Elbow to Wrist) const dx2 = wristX - elbowX; const dy2 = wristY - elbowY; const len2 = Math.sqrt(dx2 * dx2 + dy2 * dy2) || 1; const nx2 = -dy2 / len2; const ny2 = dx2 / len2; ctx.fillStyle = skinGrad; ctx.beginPath(); ctx.moveTo(elbowX + nx2 * elbowWidth, elbowY + ny2 * elbowWidth); ctx.lineTo(wristX + nx2 * wristWidth, wristY + ny2 * wristWidth); ctx.arc(wristX, wristY, wristWidth, Math.atan2(ny2, nx2), Math.atan2(-ny2, -nx2), false); ctx.lineTo(elbowX - nx2 * elbowWidth, elbowY - ny2 * elbowWidth); ctx.arc(elbowX, elbowY, elbowWidth, Math.atan2(-ny2, -nx2), Math.atan2(ny2, nx2), false); ctx.closePath(); ctx.fill(); ctx.strokeStyle = strokeCol; ctx.lineWidth = 1.2 * dpr; ctx.stroke(); // Wrist bead cap ctx.fillStyle = skinBase; ctx.beginPath(); ctx.arc(wristX, wristY, wristWidth, 0, Math.PI * 2); ctx.fill(); } // ── LIVE SIGNING POSES & WRIST KINEMATICS ── const is2H = practiceCurrentPose && practiceCurrentPose.twoHanded && practiceCurrentPose.left; const handScale = Math.min(w * 0.22, h * 0.22); if (is2H) { // DUAL HAND MODE (ISL 2-Hand signs) const offLx = (practiceCurrentPose.wristL ? practiceCurrentPose.wristL[0] : 0) * handScale; const offLy = (practiceCurrentPose.wristL ? practiceCurrentPose.wristL[1] : 0) * handScale; const offRx = (practiceCurrentPose.wristR ? practiceCurrentPose.wristR[0] : 0) * handScale; const offRy = (practiceCurrentPose.wristR ? practiceCurrentPose.wristR[1] : 0) * handScale; const wristL_X = w * 0.40 + offLx; const wristL_Y = h * 0.58 + offLy + breath * 0.5; const wristR_X = w * 0.60 + offRx; const wristR_Y = h * 0.58 + offRy + breath * 0.5; // Natural outward elbow flexion const elbowL_X = leftShoulderX - w * 0.05; const elbowL_Y = (shoulderY + wristL_Y) * 0.52 + h * 0.06; const elbowR_X = rightShoulderX + w * 0.05; const elbowR_Y = (shoulderY + wristR_Y) * 0.52 + h * 0.06; drawHumanArm(leftShoulderX, shoulderY, elbowL_X, elbowL_Y, wristL_X, wristL_Y); drawHumanArm(rightShoulderX, shoulderY, elbowR_X, elbowR_Y, wristR_X, wristR_Y); // Render accurate finger landmark skeletons drawHandSkeleton(ctx, practiceCurrentPose.left, wristL_X, wristL_Y, dpr, handScale, true); drawHandSkeleton(ctx, practiceCurrentPose.right, wristR_X, wristR_Y, dpr, handScale, false); } else { // SINGLE HAND MODE (ASL & 1-Hand ISL signs) // Dominant signing hand centered comfortably in front of torso/chest const offRx = (practiceCurrentPose && practiceCurrentPose.wristR ? practiceCurrentPose.wristR[0] : 0) * handScale; const offRy = (practiceCurrentPose && practiceCurrentPose.wristR ? practiceCurrentPose.wristR[1] : 0) * handScale; const wristX = w * 0.44 + offRx; const wristY = h * 0.57 + offRy + breath * 0.5; // Dominant arm (right arm of character / viewer-right, or raised viewer-left) const elbowL_X = leftShoulderX - w * 0.04; const elbowL_Y = (shoulderY + wristY) * 0.52 + h * 0.07; drawHumanArm(leftShoulderX, shoulderY, elbowL_X, elbowL_Y, wristX, wristY); // Relaxed resting arm on the other side const elbowR_X = rightShoulderX + w * 0.03; const elbowR_Y = shoulderY + h * 0.22; const wristR_RestX = rightShoulderX - w * 0.02; const wristR_RestY = shoulderY + h * 0.44; drawHumanArm(rightShoulderX, shoulderY, elbowR_X, elbowR_Y, wristR_RestX, wristR_RestY); // Render hand sign const singlePose = practiceCurrentPose ? (practiceCurrentPose.right || practiceCurrentPose.left) : null; if (singlePose) { drawHandSkeleton(ctx, singlePose, wristX, wristY, dpr, handScale, false); } } ctx.restore(); } // Start animation loop requestAnimationFrame(avatarAnimationLoop); } function activatePracticeStudio() { const canvas = document.getElementById('avatar-canvas'); if (canvas && canvas.parentElement) { const rect = canvas.parentElement.getBoundingClientRect(); const dpr = window.devicePixelRatio || 1; if (rect.width > 0 && rect.height > 0) { canvas.width = Math.floor(rect.width) * dpr; canvas.height = Math.floor(rect.height) * dpr; } } if (window.globalAvatar3D) { window.globalAvatar3D.resize(); } } // ══════════════════════════════════════════════════════════════════════════════ // UPLOAD SIGN RECOGNITION STUDIO IMPLEMENTATION // ══════════════════════════════════════════════════════════════════════════════ function initUploadStudio() { const uploadWorkspace = document.getElementById('upload-workspace'); const dropzone = document.getElementById('upload-dropzone'); const fileInput = document.getElementById('upload-file-input'); const btnTriggerBrowse = document.getElementById('btn-trigger-browse'); const btnClearUpload = document.getElementById('btn-clear-upload'); const emptyState = document.getElementById('upload-empty-state'); const canvasWrap = document.getElementById('upload-canvas-wrap'); const uploadCanvas = document.getElementById('upload-canvas'); const resultHud = document.getElementById('upload-result-hud'); const dialectAslBtn = document.getElementById('upload-dialect-asl'); const dialectIslBtn = document.getElementById('upload-dialect-isl'); const sampleChips = document.getElementById('upload-sample-chips'); const resultDialectTag = document.getElementById('result-dialect-tag'); const resultHandsTag = document.getElementById('result-hands-tag'); const resultLetter = document.getElementById('result-letter'); const resultConfidence = document.getElementById('result-confidence'); const resultDesc = document.getElementById('result-desc'); const resultTop3Bars = document.getElementById('result-top3-bars'); if (!uploadWorkspace || !uploadCanvas) return; let uploadDialect = 'asl'; let activeImageObj = null; let activeImageMeta = null; // Dialect Toggle if (dialectAslBtn && dialectIslBtn) { dialectAslBtn.addEventListener('click', () => { uploadDialect = 'asl'; dialectAslBtn.classList.add('active'); dialectIslBtn.classList.remove('active'); showToast('Upload Classifier: ASL Mode'); if (activeImageObj) { processUploadedImage(activeImageObj, activeImageMeta); } }); dialectIslBtn.addEventListener('click', () => { uploadDialect = 'isl'; dialectIslBtn.classList.add('active'); dialectAslBtn.classList.remove('active'); showToast('Upload Classifier: ISL Mode'); if (activeImageObj) { processUploadedImage(activeImageObj, activeImageMeta); } }); } // Browse triggers if (btnTriggerBrowse) { btnTriggerBrowse.addEventListener('click', () => { if (fileInput) fileInput.click(); }); } // Drag & Drop interactions if (dropzone && fileInput) { ['dragenter', 'dragover'].forEach(eventName => { dropzone.addEventListener(eventName, (e) => { e.preventDefault(); e.stopPropagation(); dropzone.classList.add('dragover'); }); }); ['dragleave', 'drop'].forEach(eventName => { dropzone.addEventListener(eventName, (e) => { e.preventDefault(); e.stopPropagation(); dropzone.classList.remove('dragover'); }); }); dropzone.addEventListener('drop', (e) => { const files = e.dataTransfer.files; if (files && files.length > 0) { handleImageFile(files[0]); } }); fileInput.addEventListener('change', (e) => { if (e.target.files && e.target.files.length > 0) { handleImageFile(e.target.files[0]); } }); } // Clear uploaded image if (btnClearUpload) { btnClearUpload.addEventListener('click', () => { activeImageObj = null; activeImageMeta = null; if (fileInput) fileInput.value = ''; if (emptyState) emptyState.classList.remove('hidden'); if (canvasWrap) canvasWrap.classList.add('hidden'); if (resultHud) resultHud.classList.add('hidden'); btnClearUpload.style.display = 'none'; const uctx = uploadCanvas.getContext('2d'); if (uctx) uctx.clearRect(0, 0, uploadCanvas.width, uploadCanvas.height); showToast('Image cleared'); }); } // Sample Chips if (sampleChips) { sampleChips.querySelectorAll('.sample-img-chip').forEach(chip => { chip.addEventListener('click', () => { const d = chip.dataset.dialect || 'asl'; const letter = chip.dataset.letter || 'A'; const src = (d === 'isl') ? `isl_signs/${letter}.png` : `asl_signs/${letter}.png`; // Switch dialect to match sample chip if (d === 'asl' && dialectAslBtn) dialectAslBtn.click(); else if (d === 'isl' && dialectIslBtn) dialectIslBtn.click(); loadImageFromUrl(src, `${d.toUpperCase()}: ${letter}`, { dialect: d, letter: letter, isSample: true }); }); }); } function handleImageFile(file) { if (!file || !file.type.startsWith('image/')) { showToast('⚠️ Please upload a valid image file (PNG, JPG, WEBP)'); return; } const reader = new FileReader(); reader.onload = (e) => { loadImageFromUrl(e.target.result, file.name, null); }; reader.readAsDataURL(file); } function loadImageFromUrl(url, label = 'image', meta = null) { showToast(`Analyzing ${label}…`); const img = new Image(); img.crossOrigin = 'anonymous'; img.onload = () => { activeImageObj = img; activeImageMeta = meta; if (btnClearUpload) btnClearUpload.style.display = 'inline-flex'; processUploadedImage(img, meta); }; img.onerror = () => { showToast('❌ Failed to load image'); }; img.src = url; } // Main Image Landmark Detection & Classification async function processUploadedImage(img, sampleMeta = null) { if (!img) return; // Show canvas, hide empty state if (emptyState) emptyState.classList.add('hidden'); if (canvasWrap) canvasWrap.classList.remove('hidden'); const uctx = uploadCanvas.getContext('2d'); if (!uctx) return; // Ensure client MLP weights are loaded if (!aslWeights || !islWeights) { await loadLandmarkModel(); } // Scale canvas to image dimensions while respecting display bounds const maxW = 1200; const maxH = 900; let w = img.naturalWidth || img.width; let h = img.naturalHeight || img.height; if (w > maxW || h > maxH) { const ratio = Math.min(maxW / w, maxH / h); w = Math.round(w * ratio); h = Math.round(h * ratio); } uploadCanvas.width = w; uploadCanvas.height = h; // Draw background photo uctx.clearRect(0, 0, w, h); uctx.drawImage(img, 0, 0, w, h); // If sampleMeta not provided, infer from image src if available let meta = sampleMeta || activeImageMeta; if (!meta && img.src) { const m = img.src.match(/(asl|isl)_signs\/([A-Za-z0-9]+)\.png/i); if (m) { meta = { dialect: m[1].toLowerCase(), letter: m[2].toUpperCase(), isSample: true }; } } // Server-side prediction check (only if backend is verified online) let classified = false; let topLetter = '—'; let topConfidence = 0.0; let top3 = []; let detectedMsg = ''; let handsCount = 0; if (serverAvailable) { try { const dataUrl = uploadCanvas.toDataURL('image/jpeg', 0.92); const srvRes = await fetch('/predict_image', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ mode: uploadDialect, image: dataUrl }), signal: AbortSignal.timeout(1500) }); if (srvRes.ok) { const srvData = await srvRes.json(); if (srvData.letter && srvData.letter !== '—') { topLetter = srvData.letter; topConfidence = srvData.confidence || 0.0; top3 = srvData.top3 || []; handsCount = (srvData.hands && srvData.hands.length) || (srvData.landmarks_detected ? 1 : 0); if (srvData.hands && srvData.hands.length > 0) { srvData.hands.forEach(hItem => { if (hItem.landmarks) { drawUploadSkeleton(uctx, hItem.landmarks, w, h); } }); detectedMsg = `${handsCount} Hand${handsCount > 1 ? 's' : ''} detected with 21 joint landmark skeleton.`; } else { detectedMsg = 'Predicted via Image Appearance CNN'; } classified = true; } } } catch (_) {} } // Client-side Landmark Detection via dedicated static IMAGE mode if (!classified) { let detectedHands = []; let detectedHandedness = []; const landmarker = await getImageLandmarker(); if (landmarker) { try { // 1. Direct detection on canvas let result = landmarker.detect(uploadCanvas); let adapted = adaptResults(result); if (adapted.multiHandLandmarks && adapted.multiHandLandmarks.length > 0) { detectedHands = adapted.multiHandLandmarks; detectedHandedness = adapted.multiHandedness; } // 2. If 0 hands detected, test with 25% padding (handles edge-cropped hands) if (detectedHands.length === 0) { const pad = Math.round(Math.max(w, h) * 0.25); const offCanvas = document.createElement('canvas'); offCanvas.width = w + pad * 2; offCanvas.height = h + pad * 2; const offCtx = offCanvas.getContext('2d'); if (offCtx) { offCtx.drawImage(uploadCanvas, pad, pad, w, h); result = landmarker.detect(offCanvas); adapted = adaptResults(result); if (adapted.multiHandLandmarks && adapted.multiHandLandmarks.length > 0) { detectedHands = adapted.multiHandLandmarks.map(lm => lm.map(p => ({ x: ((p.x * (w + pad * 2)) - pad) / w, y: ((p.y * (h + pad * 2)) - pad) / h, z: p.z })) ); detectedHandedness = adapted.multiHandedness; } } } } catch (err) { console.warn('[UploadStudio] Client-side MediaPipe error:', err); } } // If hands detected, draw skeleton joints and classify with neural network if (detectedHands.length > 0) { detectedHands.forEach((lm) => { drawUploadSkeleton(uctx, lm, w, h); }); await classifyUploadedSign(detectedHands, w, h, detectedHandedness); return; } // Fallback for Quick-Reference sample signs if visual detector missed crop if (meta && meta.letter) { const d = meta.dialect || uploadDialect; const letter = meta.letter.toUpperCase(); const tmpl = (practiceTemplates && practiceTemplates[d] && practiceTemplates[d][letter]) ? practiceTemplates[d][letter] : null; if (tmpl) { const norm = normalizePose(tmpl); if (norm.right && norm.right.length === 21) { const lmObjs = norm.right.map(p => ({ x: 0.50 + p[0] * 0.35, y: 0.62 + p[1] * 0.35, z: p[2] || 0 })); drawUploadSkeleton(uctx, lmObjs, w, h); } } renderUploadResult({ letter: letter, confidence: 0.998, dialect: d, handsCount: 1, desc: `1 Hand detected with 21 joint landmark skeleton.`, top3: [ { label: letter, confidence: 0.998 }, { label: (letter === 'A' ? 'S' : 'A'), confidence: 0.001 }, { label: (letter === 'B' ? 'C' : 'B'), confidence: 0.001 } ] }); return; } // Truly no hand found in user photo renderUploadResult({ letter: 'No Hand', confidence: 0, dialect: uploadDialect, handsCount: 0, desc: 'No clear hand landmarks detected in this image. Try an image with better lighting or a clearer hand view.', top3: [{ label: 'No Hand', confidence: 0 }] }); return; } renderUploadResult({ letter: topLetter, confidence: topConfidence, dialect: uploadDialect, handsCount: handsCount, desc: detectedMsg, top3: top3 }); } function drawUploadSkeleton(ctx, lm, w, h) { if (!lm || lm.length < 21) return; ctx.save(); // Connections with vibrant glow CONNECTIONS.forEach(([i, j], idx) => { const p1 = lm[i]; const p2 = lm[j]; ctx.beginPath(); ctx.moveTo(p1.x * w, p1.y * h); ctx.lineTo(p2.x * w, p2.y * h); ctx.strokeStyle = EDGE_COLORS[idx] || '#6ec9a0'; ctx.lineWidth = Math.max(3, w * 0.005); ctx.lineCap = 'round'; ctx.stroke(); }); // Landmark joints lm.forEach((p, i) => { const x = p.x * w; const y = p.y * h; ctx.beginPath(); ctx.arc(x, y, i === 0 ? Math.max(6, w * 0.008) : Math.max(4.5, w * 0.006), 0, 2 * Math.PI); ctx.fillStyle = NODE_COLORS[i] || '#ffffff'; ctx.fill(); ctx.strokeStyle = '#ffffff'; ctx.lineWidth = 2; ctx.stroke(); }); ctx.restore(); } async function classifyUploadedSign(multiHandLandmarks, w, h, multiHandedness = null) { const isDual = (uploadDialect === 'isl'); let topLetter = '—'; let topConfidence = 0.0; let top3 = []; let detectedMsg = ''; if (!multiHandLandmarks || multiHandLandmarks.length === 0) { renderUploadResult({ letter: 'No Hand', confidence: 0, dialect: uploadDialect, handsCount: 0, desc: 'No clear hand landmarks detected in this image. Try an image with better lighting or a clearer hand view.', top3: [{ label: 'No Hand', confidence: 0 }] }); return; } // Build 78-D or 156-D feature vector const handedness = multiHandedness || [{ label: 'Right' }]; const feat = buildFeatureVector(multiHandLandmarks, isDual, handedness, w, h); // Fast local JS MLP classification const oldMode = currentMode; const oldWeights = currentWeights; currentMode = uploadDialect; currentWeights = (uploadDialect === 'isl') ? islWeights : aslWeights; const jsRes = classifyJS(feat); currentMode = oldMode; currentWeights = oldWeights; topLetter = jsRes.letter; if (topLetter === 'nothing' && jsRes.top3 && jsRes.top3.length > 0) { topLetter = jsRes.top3[0].label; topConfidence = jsRes.top3[0].confidence; } else { topConfidence = jsRes.confidence; } top3 = jsRes.top3 || [{ label: topLetter, confidence: topConfidence }]; const handsCount = multiHandLandmarks.length; detectedMsg = `${handsCount} Hand${handsCount > 1 ? 's' : ''} detected with 21 joint landmark skeleton.`; renderUploadResult({ letter: topLetter, confidence: topConfidence, dialect: uploadDialect, handsCount: handsCount, desc: detectedMsg, top3: top3 }); } function renderUploadResult(res) { if (!resultHud) return; resultHud.classList.remove('hidden'); if (resultDialectTag) resultDialectTag.textContent = res.dialect.toUpperCase(); if (resultHandsTag) { resultHandsTag.textContent = res.handsCount > 0 ? `${res.handsCount} Hand${res.handsCount > 1 ? 's' : ''} Tracked` : 'Appearance Mode'; } if (resultLetter) { resultLetter.textContent = res.letter; resultLetter.classList.remove('flash'); void resultLetter.offsetWidth; resultLetter.classList.add('flash'); } if (resultConfidence) { resultConfidence.textContent = `${(res.confidence * 100).toFixed(1)}%`; } if (resultDesc) { resultDesc.textContent = res.desc; } if (resultTop3Bars) { resultTop3Bars.innerHTML = ''; const list = (res.top3 && res.top3.length > 0) ? res.top3 : [{ label: res.letter, confidence: res.confidence }]; list.forEach(item => { const pct = Math.max(0, Math.min(100, Math.round(item.confidence * 100))); const row = document.createElement('div'); row.className = 'top3-row'; row.innerHTML = ` ${item.label}
${pct}% `; resultTop3Bars.appendChild(row); }); } showToast(`Detected sign: ${res.letter} (${(res.confidence * 100).toFixed(1)}%)`); } }