RISE / rise-frontend.js
bChaukar's picture
Upload 12 files
3dcf698 verified
Raw History Blame Contribute Delete
26.4 kB
/*
* RISE Reading Practice Frontend
*
* Connects the reading interface to the FastAPI backend running in Colab.
* Features:
* - AI-generated reading passages
* - Whisper speech recognition
* - Reading accuracy and pronunciation feedback
* - Text-to-speech pronunciation playback
* - Hand gesture controls
* - Session summary
*
* IMPORTANT:
* Set window.RISE_API_BASE in your HTML before loading this file.
*/
(() => {
"use strict";
const API = (window.RISE_API_BASE || "").replace(/\/+$/, "");
const root = document.getElementById("rise-reading-app");
if (!root) {
console.error("RISE: Could not find #rise-reading-app.");
return;
}
if (!API) {
root.innerHTML = `
<p style="color:#b91c1c">
RISE API is not configured. Set window.RISE_API_BASE
to your HTTPS backend URL.
</p>`;
return;
}
// --------------------------------------------------
// READING INTERFACE
// --------------------------------------------------
root.innerHTML = `
<style>
#rise-reading-app {
font-family: system-ui, sans-serif;
color: #24324b;
max-width: 960px;
margin: 26px auto;
padding: 0 10px;
}
#rise-reading-app .rise-card {
border: 1px solid #e5e9f3;
border-radius: 24px;
padding: 24px;
margin: 16px 0;
background: #fff;
box-shadow: 0 8px 26px #465b8a10;
}
#rise-reading-app .rise-row {
display: flex;
gap: 10px;
flex-wrap: wrap;
align-items: center;
}
#rise-reading-app button {
border: 0;
border-radius: 15px;
min-height: 48px;
padding: 12px 18px;
background: #6654d9;
color: white;
font-weight: 800;
font-size: 0.98rem;
cursor: pointer;
box-shadow: 0 4px 0 #493bb4;
}
#rise-reading-app button.secondary {
background: #52728c;
box-shadow: 0 4px 0 #36536a;
}
#rise-reading-app button.danger {
background: #d35d72;
box-shadow: 0 4px 0 #a33d51;
}
#rise-reading-app button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
#rise-reading-app #rise-passage {
font-size: 1.2rem;
line-height: 1.9;
white-space: pre-wrap;
overflow-wrap: anywhere;
background: #f0f7ff;
padding: 22px;
border-radius: 18px;
border: 1px solid #e0edfa;
}
#rise-reading-app .rise-muted {
color: #64748b;
font-size: 0.92rem;
}
#rise-reading-app .rise-metrics {
display: grid;
grid-template-columns: repeat(
auto-fit,
minmax(130px, 1fr)
);
gap: 8px;
}
#rise-reading-app .rise-metric {
background: #f2efff;
border-radius: 14px;
padding: 14px;
overflow-wrap: anywhere;
}
#rise-reading-app .rise-metric b {
display: block;
font-size: 1.15rem;
}
#rise-reading-app #rise-status {
min-height: 1.5em;
}
#rise-reading-app .rise-err {
color: #b91c1c;
}
#rise-reading-app .rise-success {
color: #15803d;
}
#rise-reading-app video { max-width: 100%; border-radius: 16px; }
#rise-reading-app button:focus-visible { outline: 3px solid #f0bf4c; outline-offset: 3px; }
#rise-reading-app h2, #rise-reading-app h3 { color: #283653; }
#rise-reading-app .rise-muted { line-height: 1.6; }
@media (max-width: 600px) {
#rise-reading-app .rise-card { padding: 18px; }
#rise-reading-app #rise-passage { font-size: 1.12rem; }
#rise-reading-app button { width: 100%; }
}
</style>
<section class="rise-card">
<h2>๐Ÿ“– RISE Reading Practice</h2>
<p class="rise-muted">
AI-generated passages ยท Whisper speech recognition ยท
pronunciation feedback ยท hand gestures
</p>
<div class="rise-row">
<button id="rise-start">Start practice</button>
<button id="rise-record" disabled>
๐ŸŽ™๏ธ Start recording
</button>
<button id="rise-stop" disabled>
Stop &amp; evaluate
</button>
<button id="rise-finish" class="danger" disabled>
Finish session
</button>
</div>
<p id="rise-status" role="status" aria-live="polite">
Connecting to the RISE backend...
</p>
<p id="rise-level" class="rise-muted">
Your selected level and activity will appear here.
</p>
<div id="rise-passage">
Press Start practice to generate your reading passage.
</div>
<div id="rise-actions" class="rise-row"
style="margin-top:12px">
</div>
</section>
<section class="rise-card">
<h3>Speech evaluation</h3>
<p id="rise-transcript">
Your transcription will appear here.
</p>
<div id="rise-metrics" class="rise-metrics"></div>
<p id="rise-feedback"></p>
<div id="rise-errors"></div>
</section>
<section class="rise-card">
<h3>Hand gesture control</h3>
<p class="rise-muted">
Allow camera access to detect a hand gesture.
Your backend determines the gesture and its action.
</p>
<div class="rise-row">
<button id="rise-gesture" class="secondary" disabled>
๐Ÿ“ท Detect gesture
</button>
<span id="rise-gesture-result" aria-live="polite"></span>
</div>
</section>
`;
// --------------------------------------------------
// STATE AND HELPERS
// --------------------------------------------------
const $ = (id) => root.querySelector(`#${id}`);
let sessionId = null;
let recorder = null;
let chunks = [];
let startedAt = 0;
let stream = null;
let recordingInProgress = false;
let evaluationInProgress = false;
const activeEmail = (localStorage.getItem("riseActiveUserEmail") || "").trim().toLowerCase();
const profileKey = activeEmail ? "riseUser:" + activeEmail : null;
if (!activeEmail) {
root.innerHTML = "<section class='rise-card'><h2>Please sign in first</h2><p>Sign in to save your reading progress and streak.</p><a href='signin.html'>Go to sign in</a></section>";
return;
}
function loadUserProfile() {
let profile = profileKey ? JSON.parse(localStorage.getItem(profileKey) || "null") : null;
if (!profile) {
const starting = Math.min(5, Math.max(1, Number(localStorage.getItem("startingLevel")) || 1));
profile = { email: activeEmail, startingLevel: starting, unlockedLevel: starting, attempts: [], sessionsCompleted: 0, averageScore: 0 };
}
profile.startingLevel = Math.min(5, Math.max(1, Number(profile.startingLevel) || 1));
profile.unlockedLevel = Math.min(5, Math.max(profile.startingLevel, Number(profile.unlockedLevel) || profile.startingLevel));
if (!Array.isArray(profile.attempts)) profile.attempts = [];
profile.sessionsCompleted = Number(profile.sessionsCompleted) || 0;
return profile;
}
function saveEvaluationProgress(data) {
const profile = loadUserProfile();
const metrics = data.metrics || {};
const score = Number(metrics.overall);
profile.attempts.push({
timestamp: new Date().toISOString(),
level: data.level || "beginner",
part: data.part || "sentence",
overall: Number.isFinite(score) ? score : 0,
wordAccuracy: Number(metrics.word_accuracy) || 0,
pronunciation: Number(metrics.pronunciation) || 0,
completion: Number(metrics.completion) || 0,
wer: Number(data.wer) || 0,
passed: Boolean(data.passed)
});
// A level unlocks only after its story is passed at >=80%.
const levelNumbers = { beginner: 1, elementary: 2, intermediate: 3, upper_intermediate: 4, advanced: 5 };
const completedLevel = levelNumbers[data.level];
if (data.passed && data.part === "story" && Number(score) >= 80 && completedLevel) {
profile.unlockedLevel = Math.max(profile.unlockedLevel, Math.min(5, completedLevel + 1));
}
const scores = profile.attempts.map(a => Number(a.overall)).filter(Number.isFinite);
profile.averageScore = scores.length ? Math.round(scores.reduce((sum, n) => sum + n, 0) / scores.length * 100) / 100 : 0;
if (profileKey) localStorage.setItem(profileKey, JSON.stringify(profile));
return profile;
}
function syncServerProgress(progress) {
if (!progress) return;
const profile = loadUserProfile();
profile.unlockedLevel = Math.max(1, Math.min(5, Number(progress.unlocked_level) || 1));
profile.sessionsCompleted = Number(progress.sessions_completed) || 0;
profile.currentStreak = Number(progress.current_streak) || 0;
profile.longestStreak = Number(progress.longest_streak) || 0;
profile.lastCompletedDate = progress.last_completed_date || null;
profile.activeDates = Array.isArray(progress.active_dates) ? progress.active_dates : (profile.activeDates || []);
profile.attempts = Array.isArray(progress.attempts) ? progress.attempts : profile.attempts;
if (profileKey) localStorage.setItem(profileKey, JSON.stringify(profile));
return profile;
}
async function refreshServerProgress() {
const progress = await api(`/api/progress/${encodeURIComponent(activeEmail)}`);
return syncServerProgress(progress);
}
const setStatus = (message, isError = false) => {
const status = $("rise-status");
status.textContent = message;
status.className = isError ? "rise-err" : "";
};
const stopAudioTracks = () => {
if (stream) {
stream.getTracks().forEach((track) => track.stop());
stream = null;
}
};
const setSessionControls = (active) => {
$("rise-record").disabled = !active || recordingInProgress;
$("rise-stop").disabled = !active || !recordingInProgress;
$("rise-finish").disabled = !active || evaluationInProgress;
$("rise-gesture").disabled = !active || evaluationInProgress;
};
async function api(path, options = {}) {
const response = await fetch(`${API}${path}`, options);
const body = await response.json().catch(() => ({}));
if (!response.ok) {
throw new Error(
body.detail || `Backend request failed (${response.status})`
);
}
return body;
}
function showPassage(data) {
const level = String(data.level || "beginner")
.replaceAll("_", " ");
$("rise-level").textContent =
`Level: ${level} ยท Part: ${data.part || "sentence"} ยท ` +
`Pass mark: ${data.threshold ?? 80}%`;
$("rise-passage").textContent =
data.passage || "No passage was returned by the backend.";
}
function formatMetric(value, suffix = "%") {
if (value === undefined || value === null) {
return "N/A";
}
return `${value}${suffix}`;
}
// --------------------------------------------------
// START READING SESSION
// --------------------------------------------------
$("rise-start").addEventListener("click", async () => {
if (sessionId) {
setStatus("A session is already active.");
return;
}
$("rise-start").disabled = true;
setStatus("Starting your session and generating a passage...");
try {
/*
* practice.html should set RISE_SELECTED_LEVEL.
* If it is not set, the backend starts at Beginner.
*/
const selectedLevel =
window.RISE_SELECTED_LEVEL || "beginner";
const data = await api("/api/session/start", {
method: "POST",
headers: {
"Content-Type": "application/json"
},
body: JSON.stringify({
user_id: activeEmail,
level: selectedLevel,
part: "sentence"
})
});
if (!data.session_id) {
throw new Error("The backend did not return a session ID.");
}
sessionId = data.session_id;
if (data.progress) syncServerProgress(data.progress);
showPassage(data);
$("rise-transcript").textContent =
"Your transcription will appear here.";
$("rise-metrics").innerHTML = "";
$("rise-feedback").textContent = "";
$("rise-errors").innerHTML = "";
setStatus(
`Your session is ready. Text model: ${data.model || "loaded"}`
);
setSessionControls(true);
$("rise-start").disabled = true;
} catch (error) {
console.error("RISE session-start error:", error);
setStatus(
`Could not start the session: ${error.message}`,
true
);
$("rise-start").disabled = false;
}
});
// --------------------------------------------------
// START AUDIO RECORDING
// --------------------------------------------------
$("rise-record").addEventListener("click", async () => {
if (!sessionId || recordingInProgress || evaluationInProgress) {
return;
}
if (!navigator.mediaDevices?.getUserMedia ||
typeof MediaRecorder === "undefined") {
setStatus(
"Audio recording is not supported by this browser. " +
"Use a recent browser over HTTPS.",
true
);
return;
}
try {
stream = await navigator.mediaDevices.getUserMedia({
audio: {
channelCount: 1,
echoCancellation: true,
noiseSuppression: true
}
});
let mimeType = "";
if (MediaRecorder.isTypeSupported("audio/webm;codecs=opus")) {
mimeType = "audio/webm;codecs=opus";
} else if (MediaRecorder.isTypeSupported("audio/webm")) {
mimeType = "audio/webm";
}
recorder = mimeType
? new MediaRecorder(stream, { mimeType })
: new MediaRecorder(stream);
chunks = [];
recorder.addEventListener("dataavailable", (event) => {
if (event.data && event.data.size > 0) {
chunks.push(event.data);
}
});
recorder.start(150);
startedAt = Date.now();
recordingInProgress = true;
setSessionControls(true);
setStatus(
"Recording... Read the displayed passage aloud, " +
"then press Stop & evaluate."
);
} catch (error) {
console.error("RISE recording error:", error);
stopAudioTracks();
recordingInProgress = false;
setSessionControls(Boolean(sessionId));
setStatus(
`Could not start the microphone: ${error.message}. ` +
"Check your browser's microphone permission.",
true
);
}
});
// --------------------------------------------------
// STOP RECORDING AND EVALUATE SPEECH
// --------------------------------------------------
$("rise-stop").addEventListener("click", async () => {
if (!recorder ||
recorder.state === "inactive" ||
evaluationInProgress) {
return;
}
evaluationInProgress = true;
setSessionControls(Boolean(sessionId));
setStatus("Uploading audio and evaluating your reading with Whisper...");
const activeRecorder = recorder;
try {
const audioBlob = await new Promise((resolve, reject) => {
activeRecorder.addEventListener("stop", () => {
if (chunks.length === 0) {
reject(new Error("No audio was recorded. Please try again."));
return;
}
resolve(
new Blob(chunks, {
type: activeRecorder.mimeType || "audio/webm"
})
);
}, { once: true });
activeRecorder.stop();
});
stopAudioTracks();
recordingInProgress = false;
const elapsedSeconds = (Date.now() - startedAt) / 1000;
const formData = new FormData();
formData.append("audio", audioBlob, "reading.webm");
formData.append(
"elapsed_seconds",
String(elapsedSeconds)
);
const data = await api(
`/api/session/${encodeURIComponent(sessionId)}/evaluate`,
{
method: "POST",
body: formData
}
);
// Show the speech recognition result.
$("rise-transcript").textContent =
`Heard: ${data.transcript || "(No speech recognized)"}`;
// Show evaluation metrics.
const metrics = data.metrics || {};
const metricItems = [
["Overall score", formatMetric(metrics.overall)],
["Word accuracy", formatMetric(metrics.word_accuracy)],
["Pronunciation", formatMetric(metrics.pronunciation)],
["Completion", formatMetric(metrics.completion)],
["Word error rate", formatMetric(data.wer)],
["Speaking rate", formatMetric(metrics.speaking_rate, " wpm")]
];
$("rise-metrics").replaceChildren();
metricItems.forEach(([label, value]) => {
const card = document.createElement("div");
card.className = "rise-metric";
const name = document.createElement("span");
name.textContent = label;
const result = document.createElement("b");
result.textContent = value;
card.append(name, result);
$("rise-metrics").appendChild(card);
});
const passed = Boolean(data.passed);
saveEvaluationProgress(data);
try { await refreshServerProgress(); } catch (progressError) { console.warn("Could not refresh server progress:", progressError); }
$("rise-feedback").textContent =
`${passed ? "โœ… Passed! " : "๐Ÿ” Try again. "}` +
(data.message || "");
$("rise-feedback").className =
passed ? "rise-success" : "";
// Show detected words and pronunciation playback controls.
$("rise-errors").replaceChildren();
const events = Array.isArray(data.events) ? data.events : [];
events
.filter((event) => event.type !== "correct")
.forEach((event) => {
const row = document.createElement("div");
row.className = "rise-row";
row.style.margin = "6px 0";
const label = document.createElement("span");
label.textContent =
`${event.word || "Word"}: ${event.type || "mismatch"}` +
(event.heard ? ` (heard "${event.heard}")` : "");
const button = document.createElement("button");
button.className = "secondary";
button.textContent =
`๐Ÿ”Š Hear "${event.word || "word"}"`;
button.addEventListener("click", async () => {
try {
const audio = new Audio(
`${API}/api/pronounce/${encodeURIComponent(
event.word || ""
)}`
);
await audio.play();
} catch (error) {
setStatus(
`Could not play pronunciation: ${error.message}`,
true
);
}
});
row.append(label, button);
$("rise-errors").appendChild(row);
});
// Handle progression returned by the backend.
if (data.completed) {
if (data.progress) syncServerProgress(data.progress);
setStatus(
data.message ||
"Congratulations! You completed the reading levels."
);
$("rise-passage").textContent =
"Reading session completed. Finish the session to view your summary.";
sessionId = null;
$("rise-start").disabled = false;
setSessionControls(false);
} else if (data.next_passage) {
showPassage({
level: data.next_level,
part: data.next_part,
passage: data.next_passage,
threshold: data.threshold
});
setStatus(
data.message ||
(passed
? "Great work! Continue with the next activity."
: "You need to try this activity again.")
);
// The backend determines whether the next passage changes.
setSessionControls(true);
} else {
setStatus(
data.message ||
"Evaluation completed, but the backend did not return a next passage.",
!passed
);
setSessionControls(true);
}
} catch (error) {
console.error("RISE evaluation error:", error);
stopAudioTracks();
recordingInProgress = false;
setStatus(
`Could not evaluate the recording: ${error.message}`,
true
);
setSessionControls(Boolean(sessionId));
} finally {
evaluationInProgress = false;
if (sessionId) {
setSessionControls(true);
}
}
});
// --------------------------------------------------
// FINISH SESSION AND DISPLAY SUMMARY
// --------------------------------------------------
$("rise-finish").addEventListener("click", async () => {
if (!sessionId || recordingInProgress || evaluationInProgress) {
return;
}
const finishedSessionId = sessionId;
$("rise-finish").disabled = true;
$("rise-gesture").disabled = true;
setStatus("Preparing your session summary...");
try {
const data = await api(
`/api/session/${encodeURIComponent(finishedSessionId)}/finish`,
{
method: "POST"
}
);
if (data.progress) syncServerProgress(data.progress);
setStatus(
`Session finished. Attempts: ${data.attempts ?? "N/A"}; ` +
`average overall score: ${data.average_overall ?? "N/A"}%; ` +
`average word error rate: ${data.average_wer ?? "N/A"}%.`
);
$("rise-feedback").textContent =
JSON.stringify(data.text_generation ?? {}, null, 2);
sessionId = null;
$("rise-start").disabled = false;
setSessionControls(false);
} catch (error) {
console.error("RISE finish-session error:", error);
setStatus(
`Could not finish the session: ${error.message}`,
true
);
setSessionControls(Boolean(sessionId));
}
});
// --------------------------------------------------
// HAND GESTURE DETECTION
// --------------------------------------------------
$("rise-gesture").addEventListener("click", async () => {
if (!sessionId || evaluationInProgress || recordingInProgress) {
return;
}
if (!navigator.mediaDevices?.getUserMedia) {
setStatus(
"Camera access is not supported by this browser.",
true
);
return;
}
let cameraStream = null;
$("rise-gesture").disabled = true;
$("rise-gesture-result").textContent = "Opening camera...";
try {
cameraStream = await navigator.mediaDevices.getUserMedia({
video: {
facingMode: "user"
}
});
const video = document.createElement("video");
video.muted = true;
video.playsInline = true;
video.srcObject = cameraStream;
await video.play();
// Allow the camera to produce a frame.
await new Promise((resolve) => setTimeout(resolve, 700));
if (!video.videoWidth || !video.videoHeight) {
throw new Error("The camera did not provide a usable image.");
}
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext("2d");
if (!context) {
throw new Error("Could not capture the camera image.");
}
context.drawImage(video, 0, 0);
const imageBlob = await new Promise((resolve, reject) => {
canvas.toBlob(
(blob) => {
if (blob) {
resolve(blob);
} else {
reject(new Error("Could not encode the camera image."));
}
},
"image/jpeg",
0.85
);
});
cameraStream.getTracks().forEach((track) => track.stop());
cameraStream = null;
const formData = new FormData();
formData.append("image", imageBlob, "gesture.jpg");
const data = await api(
`/api/session/${encodeURIComponent(sessionId)}/gesture`,
{
method: "POST",
body: formData
}
);
$("rise-gesture-result").textContent =
`Detected: ${data.gesture || "unknown"} ยท ` +
`Action: ${data.action || "none"}`;
if (data.action === "level_locked") {
setStatus(data.message || "That level is locked. Complete the previous level's story with 80% or above.", true);
}
if (data.passage) {
showPassage({
level: data.level,
part: data.part,
passage: data.passage,
threshold: data.threshold || 80
});
}
if (data.ended) {
setStatus("Session ended by the hand gesture.");
sessionId = null;
$("rise-start").disabled = false;
setSessionControls(false);
} else if (data.action && data.action !== "none" && data.action !== "level_locked") {
setStatus(`Gesture applied: ${data.action}.`);
}
} catch (error) {
console.error("RISE gesture error:", error);
setStatus(
`Camera or gesture error: ${error.message}`,
true
);
$("rise-gesture-result").textContent = "Gesture detection failed.";
} finally {
if (cameraStream) {
cameraStream.getTracks().forEach((track) => track.stop());
}
$("rise-gesture").disabled = !sessionId;
}
});
// --------------------------------------------------
// CHECK BACKEND CONNECTIVITY
// --------------------------------------------------
api("/api/health")
.then((health) => {
if (!health.ok) {
throw new Error("The backend health check did not return OK.");
}
const models = health.models || {};
setStatus(
`Backend connected. Whisper: ${
models.whisper ? "ready" : "unavailable"
}; text generation: ${
models.text_generation ? "ready" : "unavailable"
}; gestures: ${
models.gesture ? "ready" : "unavailable"
}.`
);
})
.catch((error) => {
console.error("RISE health-check error:", error);
setStatus(
"Backend not reachable. Check the API URL and make sure " +
"the Colab server and Cloudflare tunnel are running.",
true
);
});
})();