Spaces:
Running
Running
Download rise-frontend.js from RISEFinalYear/RISE: direct link, hf CLI and curl.
- Browser
- Download file 26.4 kB
-
https://huggingface.co/spaces/RISEFinalYear/RISE/resolve/main/rise-frontend.js
- Command line
-
hf download hf://spaces/RISEFinalYear/RISE/rise-frontend.js
-
curl -L -o rise-frontend.js https://huggingface.co/spaces/RISEFinalYear/RISE/resolve/main/rise-frontend.js
26.4 kB
| /* | |
| * RISE Reading Practice Frontend | |
| * | |
| * Connects the reading interface to the FastAPI backend running in Colab. | |
| * Features: | |
| * - AI-generated reading passages | |
| * - Whisper speech recognition | |
| * - Reading accuracy and pronunciation feedback | |
| * - Text-to-speech pronunciation playback | |
| * - Hand gesture controls | |
| * - Session summary | |
| * | |
| * IMPORTANT: | |
| * Set window.RISE_API_BASE in your HTML before loading this file. | |
| */ | |
| (() => { | |
| "use strict"; | |
| const API = (window.RISE_API_BASE || "").replace(/\/+$/, ""); | |
| const root = document.getElementById("rise-reading-app"); | |
| if (!root) { | |
| console.error("RISE: Could not find #rise-reading-app."); | |
| return; | |
| } | |
| if (!API) { | |
| root.innerHTML = ` | |
| <p style="color:#b91c1c"> | |
| RISE API is not configured. Set window.RISE_API_BASE | |
| to your HTTPS backend URL. | |
| </p>`; | |
| return; | |
| } | |
| // -------------------------------------------------- | |
| // READING INTERFACE | |
| // -------------------------------------------------- | |
| root.innerHTML = ` | |
| <style> | |
| #rise-reading-app { | |
| font-family: system-ui, sans-serif; | |
| color: #24324b; | |
| max-width: 960px; | |
| margin: 26px auto; | |
| padding: 0 10px; | |
| } | |
| #rise-reading-app .rise-card { | |
| border: 1px solid #e5e9f3; | |
| border-radius: 24px; | |
| padding: 24px; | |
| margin: 16px 0; | |
| background: #fff; | |
| box-shadow: 0 8px 26px #465b8a10; | |
| } | |
| #rise-reading-app .rise-row { | |
| display: flex; | |
| gap: 10px; | |
| flex-wrap: wrap; | |
| align-items: center; | |
| } | |
| #rise-reading-app button { | |
| border: 0; | |
| border-radius: 15px; | |
| min-height: 48px; | |
| padding: 12px 18px; | |
| background: #6654d9; | |
| color: white; | |
| font-weight: 800; | |
| font-size: 0.98rem; | |
| cursor: pointer; | |
| box-shadow: 0 4px 0 #493bb4; | |
| } | |
| #rise-reading-app button.secondary { | |
| background: #52728c; | |
| box-shadow: 0 4px 0 #36536a; | |
| } | |
| #rise-reading-app button.danger { | |
| background: #d35d72; | |
| box-shadow: 0 4px 0 #a33d51; | |
| } | |
| #rise-reading-app button:disabled { | |
| opacity: 0.5; | |
| cursor: not-allowed; | |
| } | |
| #rise-reading-app #rise-passage { | |
| font-size: 1.2rem; | |
| line-height: 1.9; | |
| white-space: pre-wrap; | |
| overflow-wrap: anywhere; | |
| background: #f0f7ff; | |
| padding: 22px; | |
| border-radius: 18px; | |
| border: 1px solid #e0edfa; | |
| } | |
| #rise-reading-app .rise-muted { | |
| color: #64748b; | |
| font-size: 0.92rem; | |
| } | |
| #rise-reading-app .rise-metrics { | |
| display: grid; | |
| grid-template-columns: repeat( | |
| auto-fit, | |
| minmax(130px, 1fr) | |
| ); | |
| gap: 8px; | |
| } | |
| #rise-reading-app .rise-metric { | |
| background: #f2efff; | |
| border-radius: 14px; | |
| padding: 14px; | |
| overflow-wrap: anywhere; | |
| } | |
| #rise-reading-app .rise-metric b { | |
| display: block; | |
| font-size: 1.15rem; | |
| } | |
| #rise-reading-app #rise-status { | |
| min-height: 1.5em; | |
| } | |
| #rise-reading-app .rise-err { | |
| color: #b91c1c; | |
| } | |
| #rise-reading-app .rise-success { | |
| color: #15803d; | |
| } | |
| #rise-reading-app video { max-width: 100%; border-radius: 16px; } | |
| #rise-reading-app button:focus-visible { outline: 3px solid #f0bf4c; outline-offset: 3px; } | |
| #rise-reading-app h2, #rise-reading-app h3 { color: #283653; } | |
| #rise-reading-app .rise-muted { line-height: 1.6; } | |
| @media (max-width: 600px) { | |
| #rise-reading-app .rise-card { padding: 18px; } | |
| #rise-reading-app #rise-passage { font-size: 1.12rem; } | |
| #rise-reading-app button { width: 100%; } | |
| } | |
| </style> | |
| <section class="rise-card"> | |
| <h2>๐ RISE Reading Practice</h2> | |
| <p class="rise-muted"> | |
| AI-generated passages ยท Whisper speech recognition ยท | |
| pronunciation feedback ยท hand gestures | |
| </p> | |
| <div class="rise-row"> | |
| <button id="rise-start">Start practice</button> | |
| <button id="rise-record" disabled> | |
| ๐๏ธ Start recording | |
| </button> | |
| <button id="rise-stop" disabled> | |
| Stop & evaluate | |
| </button> | |
| <button id="rise-finish" class="danger" disabled> | |
| Finish session | |
| </button> | |
| </div> | |
| <p id="rise-status" role="status" aria-live="polite"> | |
| Connecting to the RISE backend... | |
| </p> | |
| <p id="rise-level" class="rise-muted"> | |
| Your selected level and activity will appear here. | |
| </p> | |
| <div id="rise-passage"> | |
| Press Start practice to generate your reading passage. | |
| </div> | |
| <div id="rise-actions" class="rise-row" | |
| style="margin-top:12px"> | |
| </div> | |
| </section> | |
| <section class="rise-card"> | |
| <h3>Speech evaluation</h3> | |
| <p id="rise-transcript"> | |
| Your transcription will appear here. | |
| </p> | |
| <div id="rise-metrics" class="rise-metrics"></div> | |
| <p id="rise-feedback"></p> | |
| <div id="rise-errors"></div> | |
| </section> | |
| <section class="rise-card"> | |
| <h3>Hand gesture control</h3> | |
| <p class="rise-muted"> | |
| Allow camera access to detect a hand gesture. | |
| Your backend determines the gesture and its action. | |
| </p> | |
| <div class="rise-row"> | |
| <button id="rise-gesture" class="secondary" disabled> | |
| ๐ท Detect gesture | |
| </button> | |
| <span id="rise-gesture-result" aria-live="polite"></span> | |
| </div> | |
| </section> | |
| `; | |
| // -------------------------------------------------- | |
| // STATE AND HELPERS | |
| // -------------------------------------------------- | |
| const $ = (id) => root.querySelector(`#${id}`); | |
| let sessionId = null; | |
| let recorder = null; | |
| let chunks = []; | |
| let startedAt = 0; | |
| let stream = null; | |
| let recordingInProgress = false; | |
| let evaluationInProgress = false; | |
| const activeEmail = (localStorage.getItem("riseActiveUserEmail") || "").trim().toLowerCase(); | |
| const profileKey = activeEmail ? "riseUser:" + activeEmail : null; | |
| if (!activeEmail) { | |
| root.innerHTML = "<section class='rise-card'><h2>Please sign in first</h2><p>Sign in to save your reading progress and streak.</p><a href='signin.html'>Go to sign in</a></section>"; | |
| return; | |
| } | |
| function loadUserProfile() { | |
| let profile = profileKey ? JSON.parse(localStorage.getItem(profileKey) || "null") : null; | |
| if (!profile) { | |
| const starting = Math.min(5, Math.max(1, Number(localStorage.getItem("startingLevel")) || 1)); | |
| profile = { email: activeEmail, startingLevel: starting, unlockedLevel: starting, attempts: [], sessionsCompleted: 0, averageScore: 0 }; | |
| } | |
| profile.startingLevel = Math.min(5, Math.max(1, Number(profile.startingLevel) || 1)); | |
| profile.unlockedLevel = Math.min(5, Math.max(profile.startingLevel, Number(profile.unlockedLevel) || profile.startingLevel)); | |
| if (!Array.isArray(profile.attempts)) profile.attempts = []; | |
| profile.sessionsCompleted = Number(profile.sessionsCompleted) || 0; | |
| return profile; | |
| } | |
| function saveEvaluationProgress(data) { | |
| const profile = loadUserProfile(); | |
| const metrics = data.metrics || {}; | |
| const score = Number(metrics.overall); | |
| profile.attempts.push({ | |
| timestamp: new Date().toISOString(), | |
| level: data.level || "beginner", | |
| part: data.part || "sentence", | |
| overall: Number.isFinite(score) ? score : 0, | |
| wordAccuracy: Number(metrics.word_accuracy) || 0, | |
| pronunciation: Number(metrics.pronunciation) || 0, | |
| completion: Number(metrics.completion) || 0, | |
| wer: Number(data.wer) || 0, | |
| passed: Boolean(data.passed) | |
| }); | |
| // A level unlocks only after its story is passed at >=80%. | |
| const levelNumbers = { beginner: 1, elementary: 2, intermediate: 3, upper_intermediate: 4, advanced: 5 }; | |
| const completedLevel = levelNumbers[data.level]; | |
| if (data.passed && data.part === "story" && Number(score) >= 80 && completedLevel) { | |
| profile.unlockedLevel = Math.max(profile.unlockedLevel, Math.min(5, completedLevel + 1)); | |
| } | |
| const scores = profile.attempts.map(a => Number(a.overall)).filter(Number.isFinite); | |
| profile.averageScore = scores.length ? Math.round(scores.reduce((sum, n) => sum + n, 0) / scores.length * 100) / 100 : 0; | |
| if (profileKey) localStorage.setItem(profileKey, JSON.stringify(profile)); | |
| return profile; | |
| } | |
| function syncServerProgress(progress) { | |
| if (!progress) return; | |
| const profile = loadUserProfile(); | |
| profile.unlockedLevel = Math.max(1, Math.min(5, Number(progress.unlocked_level) || 1)); | |
| profile.sessionsCompleted = Number(progress.sessions_completed) || 0; | |
| profile.currentStreak = Number(progress.current_streak) || 0; | |
| profile.longestStreak = Number(progress.longest_streak) || 0; | |
| profile.lastCompletedDate = progress.last_completed_date || null; | |
| profile.activeDates = Array.isArray(progress.active_dates) ? progress.active_dates : (profile.activeDates || []); | |
| profile.attempts = Array.isArray(progress.attempts) ? progress.attempts : profile.attempts; | |
| if (profileKey) localStorage.setItem(profileKey, JSON.stringify(profile)); | |
| return profile; | |
| } | |
| async function refreshServerProgress() { | |
| const progress = await api(`/api/progress/${encodeURIComponent(activeEmail)}`); | |
| return syncServerProgress(progress); | |
| } | |
| const setStatus = (message, isError = false) => { | |
| const status = $("rise-status"); | |
| status.textContent = message; | |
| status.className = isError ? "rise-err" : ""; | |
| }; | |
| const stopAudioTracks = () => { | |
| if (stream) { | |
| stream.getTracks().forEach((track) => track.stop()); | |
| stream = null; | |
| } | |
| }; | |
| const setSessionControls = (active) => { | |
| $("rise-record").disabled = !active || recordingInProgress; | |
| $("rise-stop").disabled = !active || !recordingInProgress; | |
| $("rise-finish").disabled = !active || evaluationInProgress; | |
| $("rise-gesture").disabled = !active || evaluationInProgress; | |
| }; | |
| async function api(path, options = {}) { | |
| const response = await fetch(`${API}${path}`, options); | |
| const body = await response.json().catch(() => ({})); | |
| if (!response.ok) { | |
| throw new Error( | |
| body.detail || `Backend request failed (${response.status})` | |
| ); | |
| } | |
| return body; | |
| } | |
| function showPassage(data) { | |
| const level = String(data.level || "beginner") | |
| .replaceAll("_", " "); | |
| $("rise-level").textContent = | |
| `Level: ${level} ยท Part: ${data.part || "sentence"} ยท ` + | |
| `Pass mark: ${data.threshold ?? 80}%`; | |
| $("rise-passage").textContent = | |
| data.passage || "No passage was returned by the backend."; | |
| } | |
| function formatMetric(value, suffix = "%") { | |
| if (value === undefined || value === null) { | |
| return "N/A"; | |
| } | |
| return `${value}${suffix}`; | |
| } | |
| // -------------------------------------------------- | |
| // START READING SESSION | |
| // -------------------------------------------------- | |
| $("rise-start").addEventListener("click", async () => { | |
| if (sessionId) { | |
| setStatus("A session is already active."); | |
| return; | |
| } | |
| $("rise-start").disabled = true; | |
| setStatus("Starting your session and generating a passage..."); | |
| try { | |
| /* | |
| * practice.html should set RISE_SELECTED_LEVEL. | |
| * If it is not set, the backend starts at Beginner. | |
| */ | |
| const selectedLevel = | |
| window.RISE_SELECTED_LEVEL || "beginner"; | |
| const data = await api("/api/session/start", { | |
| method: "POST", | |
| headers: { | |
| "Content-Type": "application/json" | |
| }, | |
| body: JSON.stringify({ | |
| user_id: activeEmail, | |
| level: selectedLevel, | |
| part: "sentence" | |
| }) | |
| }); | |
| if (!data.session_id) { | |
| throw new Error("The backend did not return a session ID."); | |
| } | |
| sessionId = data.session_id; | |
| if (data.progress) syncServerProgress(data.progress); | |
| showPassage(data); | |
| $("rise-transcript").textContent = | |
| "Your transcription will appear here."; | |
| $("rise-metrics").innerHTML = ""; | |
| $("rise-feedback").textContent = ""; | |
| $("rise-errors").innerHTML = ""; | |
| setStatus( | |
| `Your session is ready. Text model: ${data.model || "loaded"}` | |
| ); | |
| setSessionControls(true); | |
| $("rise-start").disabled = true; | |
| } catch (error) { | |
| console.error("RISE session-start error:", error); | |
| setStatus( | |
| `Could not start the session: ${error.message}`, | |
| true | |
| ); | |
| $("rise-start").disabled = false; | |
| } | |
| }); | |
| // -------------------------------------------------- | |
| // START AUDIO RECORDING | |
| // -------------------------------------------------- | |
| $("rise-record").addEventListener("click", async () => { | |
| if (!sessionId || recordingInProgress || evaluationInProgress) { | |
| return; | |
| } | |
| if (!navigator.mediaDevices?.getUserMedia || | |
| typeof MediaRecorder === "undefined") { | |
| setStatus( | |
| "Audio recording is not supported by this browser. " + | |
| "Use a recent browser over HTTPS.", | |
| true | |
| ); | |
| return; | |
| } | |
| try { | |
| stream = await navigator.mediaDevices.getUserMedia({ | |
| audio: { | |
| channelCount: 1, | |
| echoCancellation: true, | |
| noiseSuppression: true | |
| } | |
| }); | |
| let mimeType = ""; | |
| if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) { | |
| mimeType = "audio/webm;codecs=opus"; | |
| } else if (MediaRecorder.isTypeSupported("audio/webm")) { | |
| mimeType = "audio/webm"; | |
| } | |
| recorder = mimeType | |
| ? new MediaRecorder(stream, { mimeType }) | |
| : new MediaRecorder(stream); | |
| chunks = []; | |
| recorder.addEventListener("dataavailable", (event) => { | |
| if (event.data && event.data.size > 0) { | |
| chunks.push(event.data); | |
| } | |
| }); | |
| recorder.start(150); | |
| startedAt = Date.now(); | |
| recordingInProgress = true; | |
| setSessionControls(true); | |
| setStatus( | |
| "Recording... Read the displayed passage aloud, " + | |
| "then press Stop & evaluate." | |
| ); | |
| } catch (error) { | |
| console.error("RISE recording error:", error); | |
| stopAudioTracks(); | |
| recordingInProgress = false; | |
| setSessionControls(Boolean(sessionId)); | |
| setStatus( | |
| `Could not start the microphone: ${error.message}. ` + | |
| "Check your browser's microphone permission.", | |
| true | |
| ); | |
| } | |
| }); | |
| // -------------------------------------------------- | |
| // STOP RECORDING AND EVALUATE SPEECH | |
| // -------------------------------------------------- | |
| $("rise-stop").addEventListener("click", async () => { | |
| if (!recorder || | |
| recorder.state === "inactive" || | |
| evaluationInProgress) { | |
| return; | |
| } | |
| evaluationInProgress = true; | |
| setSessionControls(Boolean(sessionId)); | |
| setStatus("Uploading audio and evaluating your reading with Whisper..."); | |
| const activeRecorder = recorder; | |
| try { | |
| const audioBlob = await new Promise((resolve, reject) => { | |
| activeRecorder.addEventListener("stop", () => { | |
| if (chunks.length === 0) { | |
| reject(new Error("No audio was recorded. Please try again.")); | |
| return; | |
| } | |
| resolve( | |
| new Blob(chunks, { | |
| type: activeRecorder.mimeType || "audio/webm" | |
| }) | |
| ); | |
| }, { once: true }); | |
| activeRecorder.stop(); | |
| }); | |
| stopAudioTracks(); | |
| recordingInProgress = false; | |
| const elapsedSeconds = (Date.now() - startedAt) / 1000; | |
| const formData = new FormData(); | |
| formData.append("audio", audioBlob, "reading.webm"); | |
| formData.append( | |
| "elapsed_seconds", | |
| String(elapsedSeconds) | |
| ); | |
| const data = await api( | |
| `/api/session/${encodeURIComponent(sessionId)}/evaluate`, | |
| { | |
| method: "POST", | |
| body: formData | |
| } | |
| ); | |
| // Show the speech recognition result. | |
| $("rise-transcript").textContent = | |
| `Heard: ${data.transcript || "(No speech recognized)"}`; | |
| // Show evaluation metrics. | |
| const metrics = data.metrics || {}; | |
| const metricItems = [ | |
| ["Overall score", formatMetric(metrics.overall)], | |
| ["Word accuracy", formatMetric(metrics.word_accuracy)], | |
| ["Pronunciation", formatMetric(metrics.pronunciation)], | |
| ["Completion", formatMetric(metrics.completion)], | |
| ["Word error rate", formatMetric(data.wer)], | |
| ["Speaking rate", formatMetric(metrics.speaking_rate, " wpm")] | |
| ]; | |
| $("rise-metrics").replaceChildren(); | |
| metricItems.forEach(([label, value]) => { | |
| const card = document.createElement("div"); | |
| card.className = "rise-metric"; | |
| const name = document.createElement("span"); | |
| name.textContent = label; | |
| const result = document.createElement("b"); | |
| result.textContent = value; | |
| card.append(name, result); | |
| $("rise-metrics").appendChild(card); | |
| }); | |
| const passed = Boolean(data.passed); | |
| saveEvaluationProgress(data); | |
| try { await refreshServerProgress(); } catch (progressError) { console.warn("Could not refresh server progress:", progressError); } | |
| $("rise-feedback").textContent = | |
| `${passed ? "โ Passed! " : "๐ Try again. "}` + | |
| (data.message || ""); | |
| $("rise-feedback").className = | |
| passed ? "rise-success" : ""; | |
| // Show detected words and pronunciation playback controls. | |
| $("rise-errors").replaceChildren(); | |
| const events = Array.isArray(data.events) ? data.events : []; | |
| events | |
| .filter((event) => event.type !== "correct") | |
| .forEach((event) => { | |
| const row = document.createElement("div"); | |
| row.className = "rise-row"; | |
| row.style.margin = "6px 0"; | |
| const label = document.createElement("span"); | |
| label.textContent = | |
| `${event.word || "Word"}: ${event.type || "mismatch"}` + | |
| (event.heard ? ` (heard "${event.heard}")` : ""); | |
| const button = document.createElement("button"); | |
| button.className = "secondary"; | |
| button.textContent = | |
| `๐ Hear "${event.word || "word"}"`; | |
| button.addEventListener("click", async () => { | |
| try { | |
| const audio = new Audio( | |
| `${API}/api/pronounce/${encodeURIComponent( | |
| event.word || "" | |
| )}` | |
| ); | |
| await audio.play(); | |
| } catch (error) { | |
| setStatus( | |
| `Could not play pronunciation: ${error.message}`, | |
| true | |
| ); | |
| } | |
| }); | |
| row.append(label, button); | |
| $("rise-errors").appendChild(row); | |
| }); | |
| // Handle progression returned by the backend. | |
| if (data.completed) { | |
| if (data.progress) syncServerProgress(data.progress); | |
| setStatus( | |
| data.message || | |
| "Congratulations! You completed the reading levels." | |
| ); | |
| $("rise-passage").textContent = | |
| "Reading session completed. Finish the session to view your summary."; | |
| sessionId = null; | |
| $("rise-start").disabled = false; | |
| setSessionControls(false); | |
| } else if (data.next_passage) { | |
| showPassage({ | |
| level: data.next_level, | |
| part: data.next_part, | |
| passage: data.next_passage, | |
| threshold: data.threshold | |
| }); | |
| setStatus( | |
| data.message || | |
| (passed | |
| ? "Great work! Continue with the next activity." | |
| : "You need to try this activity again.") | |
| ); | |
| // The backend determines whether the next passage changes. | |
| setSessionControls(true); | |
| } else { | |
| setStatus( | |
| data.message || | |
| "Evaluation completed, but the backend did not return a next passage.", | |
| !passed | |
| ); | |
| setSessionControls(true); | |
| } | |
| } catch (error) { | |
| console.error("RISE evaluation error:", error); | |
| stopAudioTracks(); | |
| recordingInProgress = false; | |
| setStatus( | |
| `Could not evaluate the recording: ${error.message}`, | |
| true | |
| ); | |
| setSessionControls(Boolean(sessionId)); | |
| } finally { | |
| evaluationInProgress = false; | |
| if (sessionId) { | |
| setSessionControls(true); | |
| } | |
| } | |
| }); | |
| // -------------------------------------------------- | |
| // FINISH SESSION AND DISPLAY SUMMARY | |
| // -------------------------------------------------- | |
| $("rise-finish").addEventListener("click", async () => { | |
| if (!sessionId || recordingInProgress || evaluationInProgress) { | |
| return; | |
| } | |
| const finishedSessionId = sessionId; | |
| $("rise-finish").disabled = true; | |
| $("rise-gesture").disabled = true; | |
| setStatus("Preparing your session summary..."); | |
| try { | |
| const data = await api( | |
| `/api/session/${encodeURIComponent(finishedSessionId)}/finish`, | |
| { | |
| method: "POST" | |
| } | |
| ); | |
| if (data.progress) syncServerProgress(data.progress); | |
| setStatus( | |
| `Session finished. Attempts: ${data.attempts ?? "N/A"}; ` + | |
| `average overall score: ${data.average_overall ?? "N/A"}%; ` + | |
| `average word error rate: ${data.average_wer ?? "N/A"}%.` | |
| ); | |
| $("rise-feedback").textContent = | |
| JSON.stringify(data.text_generation ?? {}, null, 2); | |
| sessionId = null; | |
| $("rise-start").disabled = false; | |
| setSessionControls(false); | |
| } catch (error) { | |
| console.error("RISE finish-session error:", error); | |
| setStatus( | |
| `Could not finish the session: ${error.message}`, | |
| true | |
| ); | |
| setSessionControls(Boolean(sessionId)); | |
| } | |
| }); | |
| // -------------------------------------------------- | |
| // HAND GESTURE DETECTION | |
| // -------------------------------------------------- | |
| $("rise-gesture").addEventListener("click", async () => { | |
| if (!sessionId || evaluationInProgress || recordingInProgress) { | |
| return; | |
| } | |
| if (!navigator.mediaDevices?.getUserMedia) { | |
| setStatus( | |
| "Camera access is not supported by this browser.", | |
| true | |
| ); | |
| return; | |
| } | |
| let cameraStream = null; | |
| $("rise-gesture").disabled = true; | |
| $("rise-gesture-result").textContent = "Opening camera..."; | |
| try { | |
| cameraStream = await navigator.mediaDevices.getUserMedia({ | |
| video: { | |
| facingMode: "user" | |
| } | |
| }); | |
| const video = document.createElement("video"); | |
| video.muted = true; | |
| video.playsInline = true; | |
| video.srcObject = cameraStream; | |
| await video.play(); | |
| // Allow the camera to produce a frame. | |
| await new Promise((resolve) => setTimeout(resolve, 700)); | |
| if (!video.videoWidth || !video.videoHeight) { | |
| throw new Error("The camera did not provide a usable image."); | |
| } | |
| const canvas = document.createElement("canvas"); | |
| canvas.width = video.videoWidth; | |
| canvas.height = video.videoHeight; | |
| const context = canvas.getContext("2d"); | |
| if (!context) { | |
| throw new Error("Could not capture the camera image."); | |
| } | |
| context.drawImage(video, 0, 0); | |
| const imageBlob = await new Promise((resolve, reject) => { | |
| canvas.toBlob( | |
| (blob) => { | |
| if (blob) { | |
| resolve(blob); | |
| } else { | |
| reject(new Error("Could not encode the camera image.")); | |
| } | |
| }, | |
| "image/jpeg", | |
| 0.85 | |
| ); | |
| }); | |
| cameraStream.getTracks().forEach((track) => track.stop()); | |
| cameraStream = null; | |
| const formData = new FormData(); | |
| formData.append("image", imageBlob, "gesture.jpg"); | |
| const data = await api( | |
| `/api/session/${encodeURIComponent(sessionId)}/gesture`, | |
| { | |
| method: "POST", | |
| body: formData | |
| } | |
| ); | |
| $("rise-gesture-result").textContent = | |
| `Detected: ${data.gesture || "unknown"} ยท ` + | |
| `Action: ${data.action || "none"}`; | |
| if (data.action === "level_locked") { | |
| setStatus(data.message || "That level is locked. Complete the previous level's story with 80% or above.", true); | |
| } | |
| if (data.passage) { | |
| showPassage({ | |
| level: data.level, | |
| part: data.part, | |
| passage: data.passage, | |
| threshold: data.threshold || 80 | |
| }); | |
| } | |
| if (data.ended) { | |
| setStatus("Session ended by the hand gesture."); | |
| sessionId = null; | |
| $("rise-start").disabled = false; | |
| setSessionControls(false); | |
| } else if (data.action && data.action !== "none" && data.action !== "level_locked") { | |
| setStatus(`Gesture applied: ${data.action}.`); | |
| } | |
| } catch (error) { | |
| console.error("RISE gesture error:", error); | |
| setStatus( | |
| `Camera or gesture error: ${error.message}`, | |
| true | |
| ); | |
| $("rise-gesture-result").textContent = "Gesture detection failed."; | |
| } finally { | |
| if (cameraStream) { | |
| cameraStream.getTracks().forEach((track) => track.stop()); | |
| } | |
| $("rise-gesture").disabled = !sessionId; | |
| } | |
| }); | |
| // -------------------------------------------------- | |
| // CHECK BACKEND CONNECTIVITY | |
| // -------------------------------------------------- | |
| api("/api/health") | |
| .then((health) => { | |
| if (!health.ok) { | |
| throw new Error("The backend health check did not return OK."); | |
| } | |
| const models = health.models || {}; | |
| setStatus( | |
| `Backend connected. Whisper: ${ | |
| models.whisper ? "ready" : "unavailable" | |
| }; text generation: ${ | |
| models.text_generation ? "ready" : "unavailable" | |
| }; gestures: ${ | |
| models.gesture ? "ready" : "unavailable" | |
| }.` | |
| ); | |
| }) | |
| .catch((error) => { | |
| console.error("RISE health-check error:", error); | |
| setStatus( | |
| "Backend not reachable. Check the API URL and make sure " + | |
| "the Colab server and Cloudflare tunnel are running.", | |
| true | |
| ); | |
| }); | |
| })(); | |