/** * UniRepLKNet Speech Emotion Recognition - Client Application * Features: * - Microphone Live Recording with VU Meter * - Audio File Upload & Drag-and-Drop * - Interactive Waveform Timeline Visualizer (WaveSurfer-style with Red Playhead) * - Audio Playback & Seek Controls * - UniRepLKNet + DINOv3 ViT Emotion Classification */ document.addEventListener('DOMContentLoaded', () => { // Elements const btnThemeToggle = document.getElementById('btnThemeToggle'); const themeIcon = document.getElementById('themeIcon'); const tabBtns = document.querySelectorAll('.tab-btn'); const tabPanes = document.querySelectorAll('.tab-pane'); const btnRecord = document.getElementById('btnRecord'); const recordTimer = document.getElementById('recordTimer'); const recordStatusLabel = document.getElementById('recordStatusLabel'); const micLevelFill = document.getElementById('micLevelFill'); const dropzone = document.getElementById('dropzone'); const fileInput = document.getElementById('fileInput'); const btnPlayPause = document.getElementById('btnPlayPause'); const playIcon = document.getElementById('playIcon'); const pauseIcon = document.getElementById('pauseIcon'); const playbackTimer = document.getElementById('playbackTimer'); const waveformCanvas = document.getElementById('waveformCanvas'); const waveformBox = document.getElementById('waveformBox'); const waveformCursor = document.getElementById('waveformCursor'); const waveformStats = document.getElementById('waveformStats'); const specLoading = document.getElementById('specLoading'); const dominantCard = document.getElementById('dominantCard'); const emotionEmoji = document.getElementById('emotionEmoji'); const dominantTitle = document.getElementById('dominantTitle'); const dominantConf = document.getElementById('dominantConf'); const inferenceSpeed = document.getElementById('inferenceSpeed'); const dominantExplain = document.getElementById('dominantExplain'); const probList = document.getElementById('probList'); const audioPlayer = document.getElementById('audioPlayer'); const chipStatus = document.getElementById('chipStatus'); const statusText = document.getElementById('statusText'); // State let isRecording = false; let mediaRecorder = null; let audioChunks = []; let recordStartTime = null; let recordTimerInterval = null; let audioContext = null; let analyserNode = null; let micStream = null; let micAnimId = null; let currentAudioBlob = null; // Waveform state let currentWaveformPeaks = []; let currentAudioDuration = 0; let isSeeking = false; let playheadAnimId = null; // Emotion Metadata for UI const EMOTIONS = { angry: { label: 'Giận dữ (Angry)', emoji: '😡', color: '#ef4444' }, happy: { label: 'Vui vẻ (Happy)', emoji: '😄', color: '#10b981' }, neutral: { label: 'Trung tính (Neutral)', emoji: '😐', color: '#6b7280' }, }; // -------------------------------------------------------------------------- // 1. Theme Toggle // -------------------------------------------------------------------------- const savedTheme = localStorage.getItem('ser_theme') || 'light'; applyTheme(savedTheme); btnThemeToggle.addEventListener('click', () => { const isDark = document.body.classList.contains('dark-theme'); applyTheme(isDark ? 'light' : 'dark'); redrawCurrentWaveform(); }); function applyTheme(theme) { if (theme === 'dark') { document.body.classList.remove('light-theme'); document.body.classList.add('dark-theme'); themeIcon.textContent = '☀️'; localStorage.setItem('ser_theme', 'dark'); } else { document.body.classList.remove('dark-theme'); document.body.classList.add('light-theme'); themeIcon.textContent = '🌙'; localStorage.setItem('ser_theme', 'light'); } } // -------------------------------------------------------------------------- // 2. Tab Navigation (Record / Upload) // -------------------------------------------------------------------------- tabBtns.forEach(btn => { btn.addEventListener('click', () => { tabBtns.forEach(b => b.classList.remove('active')); tabPanes.forEach(p => p.classList.remove('active')); btn.classList.add('active'); const targetTab = btn.getAttribute('data-tab'); const targetPane = document.getElementById('pane' + targetTab.charAt(0).toUpperCase() + targetTab.slice(1)); if (targetPane) targetPane.classList.add('active'); if (isRecording) stopRecording(); }); }); // -------------------------------------------------------------------------- // 3. Microphone Recording // -------------------------------------------------------------------------- btnRecord.addEventListener('click', () => { if (!isRecording) { startRecording(); } else { stopRecording(); } }); async function startRecording() { try { micStream = await navigator.mediaDevices.getUserMedia({ audio: true }); audioContext = new (window.AudioContext || window.webkitAudioContext)(); const source = audioContext.createMediaStreamSource(micStream); analyserNode = audioContext.createAnalyser(); analyserNode.fftSize = 256; source.connect(analyserNode); mediaRecorder = new MediaRecorder(micStream); audioChunks = []; mediaRecorder.ondataavailable = e => { if (e.data.size > 0) audioChunks.push(e.data); }; mediaRecorder.onstop = async () => { const rawBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType || 'audio/webm' }); const decodedBuffer = await decodeAndRenderWaveform(rawBlob); let finalBlob = rawBlob; if (decodedBuffer) { try { finalBlob = audioBufferToWavBlob(decodedBuffer); } catch (e) { console.warn('WAV encoding failed, using raw blob:', e); } } currentAudioBlob = finalBlob; const audioUrl = URL.createObjectURL(currentAudioBlob); setupAudioPlayer(audioUrl); sendAudioForPrediction(currentAudioBlob, 'mic_recording.wav'); }; mediaRecorder.start(); isRecording = true; btnRecord.classList.add('recording'); recordStatusLabel.textContent = '🔴 Đang ghi âm... Nhấn lại nút để kết thúc'; recordStartTime = Date.now(); recordTimerInterval = setInterval(() => { const elapsed = (Date.now() - recordStartTime) / 1000; const mins = Math.floor(elapsed / 60).toString().padStart(2, '0'); const secs = (elapsed % 60).toFixed(1).padStart(4, '0'); recordTimer.textContent = `${mins}:${secs}`; }, 100); updateMicMeter(); } catch (err) { alert('Không thể truy cập Microphone: ' + err.message); recordStatusLabel.textContent = 'Lỗi truy cập Microphone'; } } function stopRecording() { if (!isRecording) return; isRecording = false; btnRecord.classList.remove('recording'); recordStatusLabel.textContent = 'Đã hoàn thành ghi âm. Đang phân tích...'; clearInterval(recordTimerInterval); if (micAnimId) cancelAnimationFrame(micAnimId); micLevelFill.style.width = '0%'; if (mediaRecorder && mediaRecorder.state !== 'inactive') { mediaRecorder.stop(); } if (micStream) { micStream.getTracks().forEach(t => t.stop()); } if (audioContext && audioContext.state !== 'closed') { audioContext.close(); } } function updateMicMeter() { if (!isRecording || !analyserNode) return; const dataArray = new Uint8Array(analyserNode.frequencyBinCount); analyserNode.getByteFrequencyData(dataArray); let sum = 0; for (let i = 0; i < dataArray.length; i++) sum += dataArray[i]; const avg = sum / dataArray.length; const level = Math.min(100, Math.round((avg / 128) * 100)); micLevelFill.style.width = `${level}%`; micAnimId = requestAnimationFrame(updateMicMeter); } // -------------------------------------------------------------------------- // 4. Upload Dropzone // -------------------------------------------------------------------------- dropzone.addEventListener('dragover', e => { e.preventDefault(); dropzone.classList.add('drag-over'); }); dropzone.addEventListener('dragleave', () => { dropzone.classList.remove('drag-over'); }); dropzone.addEventListener('drop', e => { e.preventDefault(); dropzone.classList.remove('drag-over'); if (e.dataTransfer.files.length > 0) { handleUploadedFile(e.dataTransfer.files[0]); } }); fileInput.addEventListener('change', () => { if (fileInput.files.length > 0) { handleUploadedFile(fileInput.files[0]); } }); async function handleUploadedFile(file) { if (!file.type.startsWith('audio/') && !file.name.match(/\.(wav|mp3|m4a|flac|ogg)$/i)) { alert('Vui lòng chọn file âm thanh định dạng .wav, .mp3, .m4a, .flac'); return; } currentAudioBlob = file; const url = URL.createObjectURL(file); setupAudioPlayer(url); await decodeAndRenderWaveform(file); sendAudioForPrediction(file, file.name); } // -------------------------------------------------------------------------- // 5. Audio Player Setup & Sync // -------------------------------------------------------------------------- function setupAudioPlayer(url) { audioPlayer.src = url; audioPlayer.load(); btnPlayPause.disabled = false; playIcon.style.display = 'inline'; pauseIcon.style.display = 'none'; audioPlayer.onloadedmetadata = () => { currentAudioDuration = audioPlayer.duration; playbackTimer.textContent = `00:00 / ${formatTime(currentAudioDuration)}`; redrawCurrentWaveform(); }; audioPlayer.ontimeupdate = () => { playbackTimer.textContent = `${formatTime(audioPlayer.currentTime)} / ${formatTime(audioPlayer.duration)}`; if (!isSeeking) { redrawCurrentWaveform(); } }; audioPlayer.onplay = () => { playIcon.style.display = 'none'; pauseIcon.style.display = 'inline'; startPlayheadLoop(); }; audioPlayer.onpause = () => { playIcon.style.display = 'inline'; pauseIcon.style.display = 'none'; stopPlayheadLoop(); redrawCurrentWaveform(); }; audioPlayer.onended = () => { playIcon.style.display = 'inline'; pauseIcon.style.display = 'none'; stopPlayheadLoop(); redrawCurrentWaveform(); }; } btnPlayPause.addEventListener('click', () => { if (!audioPlayer.src) return; if (audioPlayer.paused) { audioPlayer.play(); } else { audioPlayer.pause(); } }); function startPlayheadLoop() { stopPlayheadLoop(); function loop() { if (!audioPlayer.paused) { redrawCurrentWaveform(); playheadAnimId = requestAnimationFrame(loop); } } playheadAnimId = requestAnimationFrame(loop); } function stopPlayheadLoop() { if (playheadAnimId) { cancelAnimationFrame(playheadAnimId); playheadAnimId = null; } } function formatTime(secs) { if (isNaN(secs) || !isFinite(secs) || secs < 0) return '00:00'; const m = Math.floor(secs / 60).toString().padStart(2, '0'); const s = Math.floor(secs % 60).toString().padStart(2, '0'); return `${m}:${s}`; } function formatPreciseTime(secs) { if (isNaN(secs) || !isFinite(secs) || secs < 0) return '00:00.0'; const m = Math.floor(secs / 60).toString().padStart(2, '0'); const s = (secs % 60).toFixed(1).padStart(4, '0'); return `${m}:${s}`; } function audioBufferToWavBlob(buffer) { const numChannels = 1; const sampleRate = buffer.sampleRate; const bitDepth = 16; let channelData; if (buffer.numberOfChannels === 1) { channelData = buffer.getChannelData(0); } else { const left = buffer.getChannelData(0); const right = buffer.getChannelData(1); channelData = new Float32Array(left.length); for (let i = 0; i < left.length; i++) { channelData[i] = (left[i] + right[i]) / 2; } } const dataSize = channelData.length * 2; const arrayBuffer = new ArrayBuffer(44 + dataSize); const view = new DataView(arrayBuffer); function writeString(offset, string) { for (let i = 0; i < string.length; i++) { view.setUint8(offset + i, string.charCodeAt(i)); } } writeString(0, 'RIFF'); view.setUint32(4, 36 + dataSize, true); writeString(8, 'WAVE'); writeString(12, 'fmt '); view.setUint32(16, 16, true); view.setUint16(20, 1, true); // PCM format view.setUint16(22, numChannels, true); view.setUint32(24, sampleRate, true); view.setUint32(28, sampleRate * numChannels * 2, true); view.setUint16(32, numChannels * 2, true); view.setUint16(34, bitDepth, true); writeString(36, 'data'); view.setUint32(40, dataSize, true); let offset = 44; for (let i = 0; i < channelData.length; i++) { let s = Math.max(-1, Math.min(1, channelData[i])); view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true); offset += 2; } return new Blob([view], { type: 'audio/wav' }); } // -------------------------------------------------------------------------- // 6. Interactive Waveform Timeline Visualizer (Canvas) // -------------------------------------------------------------------------- async function decodeAndRenderWaveform(blob) { try { specLoading.style.display = 'flex'; const arrayBuffer = await blob.arrayBuffer(); const decodeCtx = new (window.AudioContext || window.webkitAudioContext)(); const audioBuffer = await decodeCtx.decodeAudioData(arrayBuffer); currentAudioDuration = audioBuffer.duration; const channelData = audioBuffer.getChannelData(0); // 1st channel const sampleRate = audioBuffer.sampleRate; if (waveformStats) { const srKhz = Math.round(sampleRate / 1000); const ch = audioBuffer.numberOfChannels === 1 ? 'Mono' : 'Stereo'; waveformStats.textContent = `${srKhz} kHz • ${ch} • ${(channelData.length / sampleRate).toFixed(1)}s`; } // Compute ~180 bar peaks across the duration const numBars = 180; const blockSize = Math.floor(channelData.length / numBars); const peaks = []; for (let i = 0; i < numBars; i++) { const start = i * blockSize; let maxVal = 0; for (let j = 0; j < blockSize; j += 4) { // stride 4 for speed const val = Math.abs(channelData[start + j] || 0); if (val > maxVal) maxVal = val; } peaks.push(maxVal); } // Normalize peaks (max peak ~ 0.95) const maxPeak = Math.max(...peaks, 0.01); currentWaveformPeaks = peaks.map(p => p / maxPeak); decodeCtx.close(); redrawCurrentWaveform(); return audioBuffer; } catch (err) { console.warn('Decode audio buffer failed, fallback to ambient wave:', err); return null; } finally { specLoading.style.display = 'none'; } } function resizeCanvas() { if (!waveformCanvas || !waveformBox) return; const rect = waveformBox.getBoundingClientRect(); const newWidth = Math.floor(rect.width); const newHeight = Math.floor(rect.height || 230); if (newWidth > 0 && (waveformCanvas.width !== newWidth || waveformCanvas.height !== newHeight)) { waveformCanvas.width = newWidth; waveformCanvas.height = newHeight; redrawCurrentWaveform(); } } function redrawCurrentWaveform() { const currentTime = audioPlayer ? audioPlayer.currentTime : 0; const duration = currentAudioDuration > 0 ? currentAudioDuration : 5.0; drawWaveform(currentWaveformPeaks, duration, currentTime); } function drawWaveform(peaks, duration, currentTime) { const canvas = waveformCanvas; if (!canvas) return; const ctx = canvas.getContext('2d'); const width = canvas.width; const height = canvas.height; const isDark = document.body.classList.contains('dark-theme'); ctx.clearRect(0, 0, width, height); const rulerH = 34; // Top timeline ruler height const waveH = height - rulerH; const waveMidY = rulerH + waveH / 2; const maxBarH = waveH * 0.44; // Timeline background ctx.fillStyle = isDark ? '#0d131f' : '#f8fafc'; ctx.fillRect(0, 0, width, rulerH); // Separator line between ruler and waveform ctx.strokeStyle = isDark ? '#1f293d' : '#e2e8f0'; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(0, rulerH); ctx.lineTo(width, rulerH); ctx.stroke(); // ---------------------------------------------------- // 6.1 Draw Timeline Axis & Ticks (Matching Image 2) // ---------------------------------------------------- // Determine major tick step in seconds (adaptive for narrow/mobile screens) let majorStep = 2; if (width < 450) { if (duration <= 4) majorStep = 1; else if (duration <= 8) majorStep = 2; else if (duration <= 20) majorStep = 5; else if (duration <= 40) majorStep = 10; else majorStep = 20; } else { if (duration <= 4) majorStep = 0.5; else if (duration <= 8) majorStep = 1; else if (duration <= 20) majorStep = 2; else if (duration <= 40) majorStep = 4; else if (duration <= 80) majorStep = 10; else majorStep = 20; } const numTicks = Math.ceil(duration / majorStep); ctx.font = '11px "JetBrains Mono", monospace'; ctx.textAlign = 'center'; ctx.textBaseline = 'top'; for (let t = 0; t <= duration; t += majorStep) { const x = (t / duration) * width; // Draw Time Text (e.g. 00:12, 00:14, 00:16) ctx.fillStyle = isDark ? '#94a3b8' : '#64748b'; ctx.fillText(formatTime(t), x, 6); // Major vertical tick line (height 8px) ctx.strokeStyle = isDark ? '#475569' : '#cbd5e1'; ctx.beginPath(); ctx.moveTo(x, rulerH - 8); ctx.lineTo(x, rulerH); ctx.stroke(); // Minor tick marks (between major ticks) const minorStep = majorStep / 4; for (let m = 1; m < 4; m++) { const subT = t + m * minorStep; if (subT < duration) { const subX = (subT / duration) * width; ctx.strokeStyle = isDark ? '#334155' : '#e2e8f0'; ctx.beginPath(); ctx.moveTo(subX, rulerH - 4); ctx.lineTo(subX, rulerH); ctx.stroke(); } } } // ---------------------------------------------------- // 6.2 Draw Waveform Bars // ---------------------------------------------------- const progressRatio = duration > 0 ? Math.min(1, Math.max(0, currentTime / duration)) : 0; const playheadX = progressRatio * width; // If no real audio peaks yet, draw subtle placeholder waveform const dataPeaks = peaks.length > 0 ? peaks : generateAmbientPeaks(); const numBars = dataPeaks.length; const barW = Math.max(1.8, (width / numBars) * 0.65); const barStep = width / numBars; for (let i = 0; i < numBars; i++) { const x = i * barStep + barStep * 0.15; const peak = dataPeaks[i]; const h = Math.max(3, peak * maxBarH); const topY = waveMidY - h; const botY = waveMidY + h; // Color: active played vs unplayed if (x <= playheadX && peaks.length > 0) { ctx.fillStyle = isDark ? '#818cf8' : '#4f46e5'; // Played color } else { ctx.fillStyle = isDark ? '#334155' : '#cbd5e1'; // Unplayed slate grey } // Rounded vertical bar drawRoundedBar(ctx, x, topY, barW, h * 2, barW / 2); } // ---------------------------------------------------- // 6.3 Draw Red Playhead (Vertical Line like Image 2) // ---------------------------------------------------- if (duration > 0) { // Red line down through timeline and waveform ctx.strokeStyle = '#ef4444'; ctx.lineWidth = 1.5; ctx.beginPath(); ctx.moveTo(playheadX, 0); ctx.lineTo(playheadX, height); ctx.stroke(); // Tiny red marker triangle on the timeline ruler ctx.fillStyle = '#ef4444'; ctx.beginPath(); ctx.moveTo(playheadX - 4, 0); ctx.lineTo(playheadX + 4, 0); ctx.lineTo(playheadX, 6); ctx.closePath(); ctx.fill(); } } function drawRoundedBar(ctx, x, y, width, height, radius) { radius = Math.min(radius, width / 2, height / 2); ctx.beginPath(); ctx.moveTo(x + radius, y); ctx.lineTo(x + width - radius, y); ctx.quadraticCurveTo(x + width, y, x + width, y + radius); ctx.lineTo(x + width, y + height - radius); ctx.quadraticCurveTo(x + width, y + height, x + width - radius, y + height); ctx.lineTo(x + radius, y + height); ctx.quadraticCurveTo(x, y + height, x, y + height - radius); ctx.lineTo(x, y + radius); ctx.quadraticCurveTo(x, y, x + radius, y); ctx.closePath(); ctx.fill(); } function generateAmbientPeaks() { const list = []; for (let i = 0; i < 160; i++) { const v = Math.sin(i * 0.12) * 0.35 + Math.cos(i * 0.28) * 0.25 + 0.35; list.push(Math.max(0.08, v)); } return list; } // -------------------------------------------------------------------------- // 6.4 Waveform Click & Drag Seeking Interaction // -------------------------------------------------------------------------- function seekToX(clientX) { if (!audioPlayer || currentAudioDuration <= 0) return; const rect = waveformBox.getBoundingClientRect(); const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); audioPlayer.currentTime = ratio * currentAudioDuration; playbackTimer.textContent = `${formatTime(audioPlayer.currentTime)} / ${formatTime(currentAudioDuration)}`; redrawCurrentWaveform(); } waveformBox.addEventListener('pointerdown', e => { isSeeking = true; seekToX(e.clientX); waveformBox.setPointerCapture(e.pointerId); }); waveformBox.addEventListener('pointermove', e => { const rect = waveformBox.getBoundingClientRect(); const ratio = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); const hoverSecs = ratio * (currentAudioDuration > 0 ? currentAudioDuration : 5); // Update hover cursor tooltip if (waveformCursor) { waveformCursor.style.display = 'block'; waveformCursor.style.left = `${(ratio * 100).toFixed(2)}%`; waveformCursor.textContent = formatPreciseTime(hoverSecs); } if (isSeeking) { seekToX(e.clientX); } }); waveformBox.addEventListener('pointerup', e => { if (isSeeking) { isSeeking = false; try { waveformBox.releasePointerCapture(e.pointerId); } catch (_) {} } }); waveformBox.addEventListener('pointerleave', () => { if (waveformCursor) waveformCursor.style.display = 'none'; isSeeking = false; }); // -------------------------------------------------------------------------- // 7. Prediction & API Integration // -------------------------------------------------------------------------- async function sendAudioForPrediction(blob, filename) { setStatus('Đang phân tích...', true); specLoading.style.display = 'flex'; const formData = new FormData(); formData.append('file', blob, filename); try { const res = await fetch('/api/predict', { method: 'POST', body: formData }); if (!res.ok) { const err = await res.json(); throw new Error(err.detail || 'Lỗi phân tích mô hình'); } const data = await res.json(); renderResults(data); setStatus('Sẵn sàng phân tích', false); } catch (err) { alert('Lỗi: ' + err.message); setStatus('Lỗi xử lý', false); } finally { specLoading.style.display = 'none'; } } function setStatus(text, isProcessing) { statusText.textContent = text; if (isProcessing) { chipStatus.classList.add('processing'); } else { chipStatus.classList.remove('processing'); } } function renderResults(data) { if (!data || !data.dominant) return; const dominant = data.dominant; const meta = EMOTIONS[dominant.class] || { label: dominant.class, emoji: '🎧', color: '#4f46e5' }; // Update Dominant Card if (dominantCard) dominantCard.style.borderLeftColor = meta.color; if (emotionEmoji) emotionEmoji.textContent = meta.emoji; if (dominantTitle) dominantTitle.textContent = meta.label; if (dominantConf && dominant.confidence !== undefined) { dominantConf.textContent = `${(dominant.confidence * 100).toFixed(1)}% Tin cậy`; dominantConf.style.color = meta.color; dominantConf.style.backgroundColor = `${meta.color}20`; } if (inferenceSpeed && data.latency_ms !== undefined) { inferenceSpeed.textContent = `⚡ ${data.latency_ms}ms (ViT DINOv3)`; } if (dominantExplain && data.explanation) { dominantExplain.textContent = data.explanation; } // Update Probabilities List (if element exists and data has probabilities) if (probList && Array.isArray(data.probabilities)) { probList.innerHTML = ''; data.probabilities.forEach(item => { const eMeta = EMOTIONS[item.class] || { label: item.class, emoji: '', color: '#4f46e5' }; const pct = (item.prob * 100).toFixed(1); const row = document.createElement('div'); row.className = 'prob-item'; row.innerHTML = `