| document.addEventListener('DOMContentLoaded', async () => { |
| |
| const videoElem = document.getElementById('video'); |
| const overlay = document.getElementById('overlay'); |
| const ctx = overlay.getContext('2d'); |
| const status = document.getElementById('status'); |
| const predictionSpan = document.getElementById('prediction'); |
| const confidenceBar = document.getElementById('confidenceBar'); |
| const datasetJSON = document.getElementById('datasetJSON'); |
| |
| let labels = []; |
| let recording = false; |
| let recordingLabel = null; |
| let predicting = false; |
| let knnK = 5; |
| let lastSpoken = null; |
|
|
| |
| setStatus('Initializing camera...'); |
|
|
| |
| function setStatus(s) { |
| status.textContent = `Status: ${s}`; |
| } |
|
|
| function speak(text) { |
| if ('speechSynthesis' in window && text !== lastSpoken) { |
| lastSpoken = text; |
| const msg = new SpeechSynthesisUtterance(text); |
| msg.rate = 1; |
| msg.pitch = 1; |
| speechSynthesis.cancel(); |
| speechSynthesis.speak(msg); |
| } |
| } |
|
|
| |
| function landmarksToVector(landmarks) { |
| if (!landmarks || landmarks.length === 0) return null; |
| |
| const wrist = landmarks[0]; |
| const ref = landmarks[9] || landmarks[8]; |
| const dx = ref.x - wrist.x; |
| const dy = ref.y - wrist.y; |
| const scale = Math.hypot(dx, dy) || 1; |
| |
| const vec = []; |
| for (let p of landmarks) { |
| vec.push( |
| (p.x - wrist.x) / scale, |
| (p.y - wrist.y) / scale, |
| (p.z - wrist.z) / scale |
| ); |
| } |
| return vec; |
| } |
|
|
| function euclid(a, b) { |
| let s = 0; |
| for (let i = 0; i < a.length; i++) { |
| const d = a[i] - b[i]; |
| s += d * d; |
| } |
| return Math.sqrt(s); |
| } |
|
|
| function knnPredict(vec) { |
| const all = []; |
| for (const lab of labels) { |
| for (const ex of lab.examples) { |
| all.push({ label: lab.name, dist: euclid(vec, ex) }); |
| } |
| } |
| |
| if (!all.length) return { label: null, confidence: 0 }; |
| |
| all.sort((a, b) => a.dist - b.dist); |
| const k = Math.min(knnK, all.length); |
| const top = all.slice(0, k); |
| |
| const counts = {}; |
| for (const t of top) counts[t.label] = (counts[t.label] || 0) + 1; |
| |
| let best = null, bestCount = 0; |
| for (const l in counts) { |
| if (counts[l] > bestCount) { |
| best = l; |
| bestCount = counts[l]; |
| } |
| } |
| |
| return { |
| label: best, |
| confidence: bestCount / k |
| }; |
| } |
|
|
| |
| document.getElementById('addLabel').addEventListener('click', () => { |
| const name = document.getElementById('labelInput').value.trim(); |
| if (!name) return alert('Please enter a label name'); |
| if (labels.find(l => l.name === name)) return alert('Label already exists'); |
| |
| labels.push({ name, examples: [] }); |
| renderLabels(); |
| document.getElementById('labelInput').value = ''; |
| }); |
|
|
| function renderLabels() { |
| const labelsArea = document.getElementById('labelsArea'); |
| labelsArea.innerHTML = ''; |
| |
| labels.forEach((lab, idx) => { |
| const div = document.createElement('div'); |
| div.className = 'flex items-center justify-between bg-gray-700 p-2 rounded-lg'; |
| |
| const labelBtn = document.createElement('button'); |
| labelBtn.className = `flex items-center gap-2 px-3 py-1 rounded-md transition ${ |
| recordingLabel === lab.name ? 'bg-teal-600 text-white' : 'bg-gray-600 hover:bg-gray-500' |
| }`; |
| labelBtn.innerHTML = ` |
| <i data-feather="${lab.name.length > 1 ? 'type' : 'hash'}" width="16"></i> |
| ${lab.name} <span class="text-xs opacity-75">(${lab.examples.length})</span> |
| `; |
| labelBtn.addEventListener('click', () => { |
| recordingLabel = lab.name; |
| renderLabels(); |
| }); |
| |
| const deleteBtn = document.createElement('button'); |
| deleteBtn.className = 'p-1 text-red-400 hover:text-red-300 rounded-full'; |
| deleteBtn.innerHTML = '<i data-feather="trash-2" width="16"></i>'; |
| deleteBtn.addEventListener('click', () => { |
| if (confirm(`Delete label "${lab.name}"?`)) { |
| labels.splice(idx, 1); |
| if (recordingLabel === lab.name) recordingLabel = null; |
| renderLabels(); |
| } |
| }); |
| |
| div.appendChild(labelBtn); |
| div.appendChild(deleteBtn); |
| labelsArea.appendChild(div); |
| }); |
| |
| datasetJSON.value = JSON.stringify(labels, null, 2); |
| feather.replace(); |
| } |
|
|
| |
| document.getElementById('saveBtn').addEventListener('click', () => { |
| if (labels.length === 0) return alert('No data to save'); |
| |
| const blob = new Blob([JSON.stringify(labels)], { type: 'application/json' }); |
| const url = URL.createObjectURL(blob); |
| |
| const a = document.createElement('a'); |
| a.href = url; |
| a.download = 'handspeak_dataset.json'; |
| document.body.appendChild(a); |
| a.click(); |
| document.body.removeChild(a); |
| URL.revokeObjectURL(url); |
| |
| setStatus('Dataset saved'); |
| }); |
|
|
| document.getElementById('loadBtn').addEventListener('click', () => { |
| document.getElementById('fileInput').click(); |
| }); |
|
|
| document.getElementById('fileInput').addEventListener('change', (e) => { |
| const file = e.target.files[0]; |
| if (!file) return; |
| |
| const reader = new FileReader(); |
| reader.onload = () => { |
| try { |
| const data = JSON.parse(reader.result); |
| if (!Array.isArray(data)) throw new Error('Invalid format'); |
| |
| labels = data.map(l => ({ |
| name: l.name, |
| examples: l.examples || [] |
| })); |
| |
| renderLabels(); |
| setStatus('Dataset loaded successfully'); |
| } catch (err) { |
| alert('Error loading file: ' + err.message); |
| } |
| }; |
| reader.readAsText(file); |
| }); |
|
|
| |
| document.getElementById('toggleRec').addEventListener('click', () => { |
| recording = !recording; |
| |
| if (recording && !recordingLabel) { |
| alert('Please select a label first'); |
| recording = false; |
| return; |
| } |
| |
| setStatus(recording ? `Recording examples for "${recordingLabel}"` : 'Ready'); |
| document.getElementById('toggleRec').textContent = recording ? |
| 'Stop Recording' : 'Start Recording Examples'; |
| |
| document.getElementById('toggleRec').className = recording ? |
| 'w-full bg-red-600 hover:bg-red-500 text-white px-4 py-3 rounded-lg font-medium transition' : |
| 'w-full bg-blue-600 hover:bg-blue-500 text-white px-4 py-3 rounded-lg font-medium transition'; |
| }); |
|
|
| document.getElementById('predictToggle').addEventListener('click', () => { |
| predicting = !predicting; |
| |
| if (predicting && labels.length === 0) { |
| alert('Please add some labels and examples first'); |
| predicting = false; |
| return; |
| } |
| |
| setStatus(predicting ? 'Predicting...' : 'Ready'); |
| document.getElementById('predictToggle').textContent = predicting ? |
| 'Stop Predicting' : 'Start Predicting'; |
| |
| document.getElementById('predictToggle').className = predicting ? |
| 'w-full bg-purple-700 hover:bg-purple-600 text-white px-4 py-3 rounded-lg font-medium transition' : |
| 'w-full bg-purple-600 hover:bg-purple-500 text-white px-4 py-3 rounded-lg font-medium transition'; |
| }); |
|
|
| |
| const hands = new Hands({ |
| locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}` |
| }); |
| |
| hands.setOptions({ |
| maxNumHands: 1, |
| modelComplexity: 1, |
| minDetectionConfidence: 0.7, |
| minTrackingConfidence: 0.6 |
| }); |
| |
| hands.onResults(onResults); |
| |
| const camera = new Camera(videoElem, { |
| onFrame: async () => { |
| await hands.send({ image: videoElem }); |
| }, |
| width: 640, |
| height: 480 |
| }); |
| |
| camera.start(); |
| |
| |
| function onResults(results) { |
| ctx.save(); |
| ctx.clearRect(0, 0, overlay.width, overlay.height); |
| ctx.drawImage(results.image, 0, 0, overlay.width, overlay.height); |
| |
| if (results.multiHandLandmarks) { |
| for (const landmarks of results.multiHandLandmarks) { |
| drawConnectors(ctx, landmarks, HAND_CONNECTIONS, { |
| color: '#10B981', |
| lineWidth: 4 |
| }); |
| |
| drawLandmarks(ctx, landmarks, { |
| color: '#F59E0B', |
| lineWidth: 2, |
| radius: (data) => { |
| return lerp(data.from.z, -0.15, 0.1, 5, 1); |
| } |
| }); |
| } |
| } |
| |
| ctx.restore(); |
| |
| |