bChaukar commited on
Commit
9a71be7
·
verified ·
1 Parent(s): 381c486

Update rise_colab_api.py

Browse files
Files changed (1) hide show
  1. rise_colab_api.py +6 -521
rise_colab_api.py CHANGED
@@ -1,6 +1,6 @@
1
  """RISE reading-assistant API for a Google Colab backend.
2
 
3
- Uses Qwen2.5-1.5B-Instruct (FLAN-T5-base fallback), Whisper Small,
4
  gTTS pronunciation audio, and MediaPipe Hand Landmarker.
5
  No notebook-only output.eval_js or IPython.display calls are used in API routes.
6
  """
@@ -29,12 +29,8 @@ from fastapi.middleware.cors import CORSMiddleware
29
  from fastapi.responses import StreamingResponse
30
  from pydantic import BaseModel
31
  from gtts import gTTS
32
- from transformers import (
33
- pipeline as hf_pipeline,
34
- AutoTokenizer,
35
- AutoModelForSeq2SeqLM,
36
- AutoModelForCausalLM,
37
- )
38
 
39
  # --------------------------- Curriculum/configuration -----------------------
40
  ADVANCE_THRESHOLD = 80.0
@@ -83,11 +79,8 @@ FALLBACK = {
83
 
84
  # --------------------------- Model globals ----------------------------------
85
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
86
- genai_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
87
- GENAI_MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
88
- GENAI_ARCHITECTURE = "causal"
89
- genai_tokenizer = None
90
- genai_model = None
91
  speech_to_text = None
92
  _hand_landmarker = None
93
  model_status: dict[str, Any] = {"whisper": False, "text_generation": False, "gesture": False, "text_model": None, "errors": []}
@@ -100,512 +93,4 @@ _CONTRACTIONS = {
100
  "we're": "we are", "they're": "they are", "you're": "you are",
101
  "isn't": "is not", "aren't": "are not", "wasn't": "was not", "weren't": "were not",
102
  "don't": "do not", "doesn't": "does not", "didn't": "did not", "can't": "cannot",
103
- "couldn't": "could not", "won't": "will not", "wouldn't": "would not", "there's": "there is",
104
- "hadn't": "had not", "haven't": "have not", "hasn't": "has not",
105
- }
106
-
107
- def normalize_text(text: str) -> str:
108
- text = (text or "").lower().strip()
109
- for contraction, expanded in _CONTRACTIONS.items():
110
- text = text.replace(contraction, expanded)
111
- return re.sub(r"\s+", " ", re.sub(r"[^a-z\s]", " ", text)).strip()
112
-
113
- def normalize_word(word: str) -> str:
114
- return re.sub(r"[^a-z]", "", word.lower())
115
-
116
- def _count_sentences(text: str) -> int:
117
- return len([x for x in re.split(r"(?<=[.!?])\s+", text.strip()) if x.strip()])
118
-
119
- def _clean_passage(text: str, part: str) -> str:
120
- text = re.sub(r"^\s*(passage|story|paragraph|sentences)\s*:\s*", "", str(text), flags=re.I)
121
- text = text.strip().strip('"“”')
122
- paragraphs = []
123
- for para in re.split(r"\n\s*\n", text):
124
- para = re.sub(r"\s+", " ", para).strip()
125
- para = re.sub(r"^(?:[-*•]|\d+[.)])\s+", "", para)
126
- if not para:
127
- continue
128
- sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", para) if s.strip()]
129
- para = " ".join(sentences)
130
- if para and para[-1] not in ".!?":
131
- para += "."
132
- if para:
133
- paragraphs.append(para)
134
- if part == "sentence":
135
- sentences = []
136
- for para in paragraphs:
137
- sentences.extend([s.strip() for s in re.split(r"(?<=[.!?])\s+", para) if s.strip()])
138
- return " ".join(sentences[:NUM_SENTENCES])
139
- if part == "paragraph":
140
- return " ".join(paragraphs)
141
- return "\n\n".join(paragraphs)
142
-
143
- # --------------------------- Loading models ---------------------------------
144
- def load_models() -> None:
145
- global speech_to_text, genai_tokenizer, genai_model, GENAI_MODEL_NAME, GENAI_ARCHITECTURE, _hand_landmarker
146
- if speech_to_text is None:
147
- try:
148
- speech_to_text = hf_pipeline(
149
- "automatic-speech-recognition", model="openai/whisper-small",
150
- device=0 if torch.cuda.is_available() else -1,
151
- torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
152
- chunk_length_s=30, stride_length_s=5,
153
- generate_kwargs={"language": "english", "task": "transcribe"},
154
- )
155
- model_status["whisper"] = True
156
- except Exception as exc:
157
- model_status["errors"].append(f"Whisper load failed: {exc}")
158
- if genai_model is None:
159
- try:
160
- GENAI_MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
161
- GENAI_ARCHITECTURE = "causal"
162
- genai_tokenizer = AutoTokenizer.from_pretrained(GENAI_MODEL_NAME)
163
- genai_model = AutoModelForCausalLM.from_pretrained(
164
- GENAI_MODEL_NAME, torch_dtype=genai_dtype, low_cpu_mem_usage=True
165
- ).to(device)
166
- except Exception as primary_exc:
167
- model_status["errors"].append(f"Qwen could not load; trying FLAN-T5-base: {primary_exc}")
168
- try:
169
- if genai_model is not None:
170
- del genai_model
171
- GENAI_MODEL_NAME = "google/flan-t5-base"
172
- GENAI_ARCHITECTURE = "seq2seq"
173
- genai_tokenizer = AutoTokenizer.from_pretrained(GENAI_MODEL_NAME)
174
- genai_model = AutoModelForSeq2SeqLM.from_pretrained(
175
- GENAI_MODEL_NAME, torch_dtype=genai_dtype, low_cpu_mem_usage=True
176
- ).to(device)
177
- except Exception as fallback_exc:
178
- model_status["errors"].append(f"FLAN-T5 fallback failed: {fallback_exc}")
179
- genai_model = None
180
- genai_tokenizer = None
181
- if genai_model is not None:
182
- genai_model.eval()
183
- model_status["text_generation"] = True
184
- model_status["text_model"] = GENAI_MODEL_NAME
185
- try:
186
- import mediapipe as mp
187
- from mediapipe.tasks import python as mp_python
188
- from mediapipe.tasks.python import vision as mp_vision
189
- model_path = Path("/content/hand_landmarker.task")
190
- if not model_path.exists():
191
- model_path.parent.mkdir(parents=True, exist_ok=True)
192
- url = ("https://storage.googleapis.com/mediapipe-models/hand_landmarker/"
193
- "hand_landmarker/float16/1/hand_landmarker.task")
194
- urllib.request.urlretrieve(url, str(model_path))
195
- options = mp_vision.HandLandmarkerOptions(
196
- base_options=mp_python.BaseOptions(model_asset_path=str(model_path)),
197
- num_hands=1, min_hand_detection_confidence=0.7,
198
- min_hand_presence_confidence=0.7, min_tracking_confidence=0.7,
199
- )
200
- _hand_landmarker = mp_vision.HandLandmarker.create_from_options(options)
201
- model_status["gesture"] = True
202
- except Exception as exc:
203
- model_status["errors"].append(f"MediaPipe gesture model unavailable: {exc}")
204
-
205
- def _model_generate(prompt: str, max_new_tokens: int = 180) -> str:
206
- if genai_model is None or genai_tokenizer is None:
207
- raise RuntimeError("Text generation model is not available")
208
- if GENAI_ARCHITECTURE == "causal":
209
- messages = [
210
- {"role": "system", "content": "You write clear, grammatical English reading passages. Output only the requested reading text."},
211
- {"role": "user", "content": prompt},
212
- ]
213
- rendered = genai_tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
214
- encoded = genai_tokenizer(rendered, return_tensors="pt", truncation=True, max_length=2048).to(device)
215
- with torch.inference_mode():
216
- output_ids = genai_model.generate(
217
- **encoded, max_new_tokens=max_new_tokens, do_sample=True,
218
- temperature=0.65, top_p=0.9, repetition_penalty=1.12,
219
- pad_token_id=genai_tokenizer.eos_token_id,
220
- )
221
- new_ids = output_ids[0][encoded["input_ids"].shape[1]:]
222
- return genai_tokenizer.decode(new_ids, skip_special_tokens=True).strip()
223
- encoded = genai_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512).to(device)
224
- with torch.inference_mode():
225
- output_ids = genai_model.generate(
226
- **encoded, max_new_tokens=max_new_tokens, do_sample=True,
227
- temperature=0.7, top_p=0.9, repetition_penalty=1.12,
228
- )
229
- return genai_tokenizer.decode(output_ids[0], skip_special_tokens=True).strip()
230
-
231
- def _generate_paragraph(level: str, target_words: int, context: str = "") -> str:
232
- prompt = (
233
- "Write ONE complete English paragraph for a reading-practice app. "
234
- f"Difficulty: {level.replace('_', ' ')}. {LEVEL_GUIDANCE[level]} "
235
- f"Length: about {target_words} words. Use at least four complete sentences, "
236
- "one coherent topic, no title, bullets, or numbering. "
237
- + (f"Story context: {context}. " if context else "") + "Return only the paragraph."
238
- )
239
- last = ""
240
- for _ in range(3):
241
- last = _clean_passage(_model_generate(prompt, max_new_tokens=min(512, max(180, int(target_words * 2.4)))), "paragraph")
242
- if _count_sentences(last) >= 4 and len(last.split()) >= max(25, int(target_words * 0.65)):
243
- return last
244
- prompt += " Make the paragraph longer and include at least four complete sentences."
245
- return last
246
-
247
- def generate_passage(level: str, part: str) -> str:
248
- started = time.time()
249
- spec = LEVEL_SPECS[level]
250
- used_fallback = False
251
- min_words, max_words = spec["sentence_words"]
252
- target_words = NUM_SENTENCES * ((min_words + max_words) // 2) if part == "sentence" else (
253
- spec["paragraph_words"] if part == "paragraph" else spec["story_paragraphs"] * spec["story_words_per_paragraph"]
254
- )
255
- try:
256
- if part == "sentence":
257
- prompt = (
258
- f"Write exactly three original English sentences for reading practice at {level.replace('_',' ')} level. "
259
- f"{LEVEL_GUIDANCE[level]} Each sentence must contain {min_words}-{max_words} words. "
260
- "Put one complete sentence per line. No title or explanation."
261
- )
262
- passage = ""
263
- for _ in range(3):
264
- candidate = _clean_passage(_model_generate(prompt, 160), "sentence")
265
- sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", candidate) if s.strip()]
266
- if len(sentences) == 3 and all(min_words <= len(s.split()) <= max_words for s in sentences):
267
- passage = " ".join(sentences)
268
- break
269
- prompt += f" Try again: exactly 3 sentences, each {min_words}-{max_words} words."
270
- if not passage:
271
- eligible = [x for x in FALLBACK[level]["sentence"][0].split(". ") if x.strip()]
272
- passage = " ".join((eligible * 3)[:3])
273
- if not passage.endswith("."):
274
- passage += "."
275
- used_fallback = True
276
- elif part == "paragraph":
277
- passage = _generate_paragraph(level, spec["paragraph_words"])
278
- if _count_sentences(passage) < 4:
279
- passage = FALLBACK[level]["paragraph"][0]
280
- used_fallback = True
281
- else:
282
- paras = []
283
- context = "Introduce the main character, setting, and situation."
284
- for i in range(spec["story_paragraphs"]):
285
- if i == spec["story_paragraphs"] - 1:
286
- context = "Continue the same story and give it a clear, satisfying ending. Previous story: " + " ".join(paras)[-900:]
287
- elif paras:
288
- context = "Continue the same story without repeating. Previous story: " + " ".join(paras)[-900:]
289
- para = _generate_paragraph(level, spec["story_words_per_paragraph"], context)
290
- if _count_sentences(para) < 3:
291
- raise ValueError("Story paragraph was too short")
292
- paras.append(para)
293
- passage = "\n\n".join(paras)
294
- except Exception as exc:
295
- used_fallback = True
296
- passage = FALLBACK[level][part][0]
297
- print(f"Passage generation fallback for {level}/{part}: {exc}")
298
- if not passage.strip():
299
- passage = FALLBACK[level][part][0]
300
- used_fallback = True
301
- paragraphs = [p for p in re.split(r"\n\s*\n", passage.strip()) if p.strip()]
302
- observed = "story" if len(paragraphs) >= 2 else ("sentence" if _count_sentences(passage) <= 3 else "paragraph")
303
- record = {
304
- "level": level, "requested_format": part, "observed_format": observed,
305
- "word_count": len(passage.split()), "sentence_count": _count_sentences(passage),
306
- "paragraph_count": len(paragraphs), "target_words": target_words,
307
- "generation_seconds": round(time.time() - started, 2), "used_fallback": used_fallback,
308
- "generation_method": GENAI_MODEL_NAME if not used_fallback else "reviewed_fallback",
309
- "length_compliant": len(passage.split()) >= target_words * 0.65,
310
- "structure_compliant": (part == "sentence" and _count_sentences(passage) == 3) or (part == "paragraph" and len(paragraphs) == 1 and _count_sentences(passage) >= 4) or (part == "story" and len(paragraphs) >= 2),
311
- }
312
- generation_records.append(record)
313
- return passage
314
-
315
- # --------------------------- Speech evaluation ------------------------------
316
- def word_similarity(a: str, b: str) -> float:
317
- a, b = normalize_word(a), normalize_word(b)
318
- if not a or not b:
319
- return 0.0
320
- if a == b:
321
- return 1.0
322
- return SequenceMatcher(None, a, b).ratio()
323
-
324
- def build_events(expected_words: list[str], spoken_words: list[str]) -> list[dict[str, Any]]:
325
- events = []
326
- matcher = SequenceMatcher(None, expected_words, spoken_words, autojunk=False)
327
- for tag, i1, i2, j1, j2 in matcher.get_opcodes():
328
- if tag == "equal":
329
- for w in expected_words[i1:i2]:
330
- events.append({"word": w, "heard": w, "type": "correct", "similarity": 1.0, "attempted": True})
331
- elif tag == "replace":
332
- exp = expected_words[i1:i2]
333
- said = spoken_words[j1:j2]
334
- for i, word in enumerate(exp):
335
- heard = said[i] if i < len(said) else ""
336
- sim = word_similarity(word, heard)
337
- kind = "correct" if sim >= 0.80 else ("hesitation" if sim >= 0.50 else ("mismatch" if heard else "skipped"))
338
- events.append({"word": word, "heard": heard, "type": kind, "similarity": sim, "attempted": bool(heard)})
339
- elif tag == "delete":
340
- for word in expected_words[i1:i2]:
341
- events.append({"word": word, "heard": "", "type": "skipped", "similarity": 0.0, "attempted": False})
342
- # Extra spoken words are represented in WER; no expected word is assigned to them.
343
- return events
344
-
345
- def calculate_wer(reference: str, hypothesis: str) -> float:
346
- ref = normalize_text(reference).split()
347
- hyp = normalize_text(hypothesis).split()
348
- if not ref:
349
- return 0.0
350
- prev = list(range(len(hyp) + 1))
351
- for i, rw in enumerate(ref, 1):
352
- curr = [i]
353
- for j, hw in enumerate(hyp, 1):
354
- curr.append(prev[j-1] if rw == hw else 1 + min(prev[j-1], prev[j], curr[j-1]))
355
- prev = curr
356
- return round(prev[-1] / len(ref) * 100, 2)
357
-
358
- def evaluate(expected_words: list[str], events: list[dict[str, Any]], spoken_text: str, elapsed_secs: float) -> dict[str, Any]:
359
- total = len(expected_words)
360
- correct = sum(e["type"] == "correct" for e in events)
361
- hes = sum(e["type"] == "hesitation" for e in events)
362
- skipped = sum(e["type"] == "skipped" for e in events)
363
- mismatches = sum(e["type"] == "mismatch" for e in events)
364
- accuracy = correct / total * 100 if total else 0.0
365
- sims = [e["similarity"] for e in events if e.get("heard")]
366
- pronunciation = float(np.mean(sims)) * 100 if sims else 0.0
367
- attempted = correct + mismatches + hes
368
- completion = min(attempted / total * 100, 100.0) if total else 0.0
369
- rate = len(normalize_text(spoken_text).split()) / (elapsed_secs / 60) if elapsed_secs > 0 else 0.0
370
- overall = 0.50 * accuracy + 0.30 * pronunciation + 0.20 * completion
371
- return {"word_accuracy": round(accuracy, 2), "pronunciation": round(pronunciation, 2), "completion": round(completion, 2), "hesitations": hes, "skipped": skipped, "mismatches": mismatches, "speaking_rate": round(rate, 2), "overall": round(overall, 2)}
372
-
373
- # --------------------------- MediaPipe gesture detection --------------------
374
- def _angle(a, b, c) -> float:
375
- a, b, c = np.array([a.x, a.y]), np.array([b.x, b.y]), np.array([c.x, c.y])
376
- ba, bc = a - b, c - b
377
- cos = np.dot(ba, bc) / (np.linalg.norm(ba) * np.linalg.norm(bc) + 1e-8)
378
- return float(np.degrees(np.arccos(np.clip(cos, -1.0, 1.0))))
379
-
380
- def _finger_states(landmarks) -> list[int]:
381
- wrist, index_mcp, pinky_mcp = landmarks[0], landmarks[5], landmarks[17]
382
- palm_width = np.sqrt((index_mcp.x - pinky_mcp.x) ** 2 + (index_mcp.y - pinky_mcp.y) ** 2)
383
- palm_cx = sum(landmarks[i].x for i in [0, 5, 9, 13, 17]) / 5
384
- palm_cy = sum(landmarks[i].y for i in [0, 5, 9, 13, 17]) / 5
385
- thumb_angle = _angle(landmarks[2], landmarks[3], landmarks[4])
386
- thumb_dist = np.sqrt((landmarks[4].x - palm_cx) ** 2 + (landmarks[4].y - palm_cy) ** 2)
387
- fingers = [1 if thumb_angle > 145 and thumb_dist > palm_width * 0.65 else 0]
388
- for mcp, pip, tip in [(5, 6, 8), (9, 10, 12), (13, 14, 16), (17, 18, 20)]:
389
- angle = _angle(landmarks[mcp], landmarks[pip], landmarks[tip])
390
- tip_dist = np.sqrt((landmarks[tip].x - wrist.x) ** 2 + (landmarks[tip].y - wrist.y) ** 2)
391
- pip_dist = np.sqrt((landmarks[pip].x - wrist.x) ** 2 + (landmarks[pip].y - wrist.y) ** 2)
392
- fingers.append(1 if angle > 155 and tip_dist > pip_dist else 0)
393
- return fingers
394
-
395
- def _recognize_gesture(fingers: list[int], landmarks) -> str:
396
- if fingers == [1, 1, 1, 1, 1]: return "LEVEL_5"
397
- if fingers == [0, 1, 1, 1, 1]: return "LEVEL_4"
398
- if fingers == [0, 1, 1, 1, 0]: return "LEVEL_3"
399
- if fingers == [0, 1, 1, 0, 0]: return "LEVEL_2"
400
- if fingers == [0, 1, 0, 0, 0]: return "LEVEL_1"
401
- if fingers == [1, 0, 0, 0, 0]: return "HOME" if landmarks[4].y < landmarks[0].y else "PAUSE"
402
- return "UNKNOWN"
403
-
404
- def detect_gesture_from_bytes(raw: bytes) -> str:
405
- if _hand_landmarker is None:
406
- raise HTTPException(status_code=503, detail="Gesture detector is not available. Check /api/health.")
407
- arr = np.frombuffer(raw, dtype=np.uint8)
408
- frame = cv2.imdecode(arr, cv2.IMREAD_COLOR)
409
- if frame is None:
410
- raise HTTPException(status_code=400, detail="Could not decode the camera image.")
411
- rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
412
- import mediapipe as mp
413
- mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
414
- result = _hand_landmarker.detect(mp_image)
415
- if not result.hand_landmarks:
416
- return "UNKNOWN"
417
- landmarks = result.hand_landmarks[0]
418
- return _recognize_gesture(_finger_states(landmarks), landmarks)
419
-
420
- # --------------------------- API application --------------------------------
421
- app = FastAPI(title="RISE Reading Assistant API", version="1.0.0")
422
- app.add_middleware(
423
- CORSMiddleware, allow_origins=["*"], allow_credentials=False,
424
- allow_methods=["*"], allow_headers=["*"],
425
- )
426
-
427
- @app.on_event("startup")
428
- def _startup():
429
- print("Loading RISE models. This may take several minutes on the first run...")
430
- load_models()
431
- print("RISE model status:", model_status)
432
-
433
- @app.get("/api/health")
434
- def health():
435
- return {"ok": True, "service": "RISE", "models": model_status, "threshold": ADVANCE_THRESHOLD}
436
-
437
- class StartRequest(BaseModel):
438
- level: str = "beginner"
439
- part: str = "sentence"
440
-
441
- @app.post("/api/session/start")
442
- def start_session(request: StartRequest):
443
- if not model_status.get("text_generation"):
444
- raise HTTPException(status_code=503, detail="Text generation model is not ready. Check /api/health and server logs.")
445
- level = request.level if request.level in LEVELS else LEVELS[0]
446
- part = request.part if request.part in PARTS else PARTS[0]
447
- session_id = str(uuid.uuid4())
448
- passage = generate_passage(level, part)
449
- sessions[session_id] = {"id": session_id, "level": level, "part": part, "passage": passage, "history": [], "ended": False, "created_at": time.time()}
450
- return {"session_id": session_id, "level": level, "part": part, "passage": passage, "threshold": ADVANCE_THRESHOLD, "model": GENAI_MODEL_NAME}
451
-
452
- @app.get("/api/session/{session_id}")
453
- def get_session(session_id: str):
454
- state = sessions.get(session_id)
455
- if not state:
456
- raise HTTPException(status_code=404, detail="Session not found. Start a new session.")
457
- return {k: v for k, v in state.items() if k != "history"} | {"attempts": len(state["history"])}
458
-
459
- @app.post("/api/session/{session_id}/evaluate")
460
- async def evaluate_audio(session_id: str, audio: UploadFile = File(...), elapsed_seconds: float = Form(0.0)):
461
- state = sessions.get(session_id)
462
- if not state:
463
- raise HTTPException(status_code=404, detail="Session not found. Start a new session.")
464
- if state["ended"]:
465
- raise HTTPException(status_code=400, detail="This session has ended. Start a new session.")
466
- if not model_status.get("whisper") or speech_to_text is None:
467
- raise HTTPException(status_code=503, detail="Whisper is not ready. Check /api/health.")
468
- raw = await audio.read()
469
- if not raw:
470
- raise HTTPException(status_code=400, detail="The uploaded audio is empty.")
471
- suffix = ".webm" if "webm" in (audio.content_type or "") or (audio.filename or "").endswith(".webm") else (Path(audio.filename or "recording.wav").suffix or ".wav")
472
- try:
473
- with tempfile.TemporaryDirectory() as tmp:
474
- in_path, wav_path = Path(tmp) / f"input{suffix}", Path(tmp) / "audio.wav"
475
- in_path.write_bytes(raw)
476
- proc = subprocess.run(
477
- ["ffmpeg", "-y", "-loglevel", "error", "-i", str(in_path), "-ar", str(SAMPLE_RATE), "-ac", "1", "-sample_fmt", "s16", str(wav_path)],
478
- capture_output=True, text=True, timeout=90,
479
- )
480
- if proc.returncode != 0 or not wav_path.exists():
481
- raise ValueError(proc.stderr[-800:] or "ffmpeg could not decode the audio")
482
- samples, sr = sf.read(str(wav_path), dtype="float32")
483
- except Exception as exc:
484
- raise HTTPException(status_code=400, detail=f"Could not decode recording. Please record again. Details: {str(exc)[:300]}")
485
- if samples.ndim > 1:
486
- samples = samples.mean(axis=1)
487
- if len(samples) < SAMPLE_RATE // 2:
488
- raise HTTPException(status_code=400, detail="Recording is too short. Please read aloud for at least half a second.")
489
- peak = float(np.max(np.abs(samples))) if len(samples) else 0.0
490
- if peak > 0.001:
491
- samples = samples * min(0.7 / peak, 3.0)
492
- duration = elapsed_seconds if elapsed_seconds > 0 else len(samples) / SAMPLE_RATE
493
- try:
494
- transcript = speech_to_text({"array": samples, "sampling_rate": SAMPLE_RATE})["text"].strip()
495
- except Exception as exc:
496
- raise HTTPException(status_code=500, detail=f"Whisper transcription failed: {str(exc)[:250]}")
497
- if not normalize_text(transcript):
498
- raise HTTPException(status_code=400, detail="No speech was detected. Check microphone permissions and try again.")
499
- expected_words = normalize_text(state["passage"]).split()
500
- spoken_words = normalize_text(transcript).split()
501
- events = build_events(expected_words, spoken_words)
502
- metrics = evaluate(expected_words, events, transcript, duration)
503
- wer = calculate_wer(state["passage"], transcript)
504
- pronunciation_words = list(dict.fromkeys(e["word"] for e in events if e["type"] in {"hesitation", "mismatch", "skipped"}))
505
- attempt = {"level": state["level"], "part": state["part"], "transcript": transcript, "metrics": metrics, "wer": wer, "timestamp": time.time()}
506
- state["history"].append(attempt)
507
- passed = metrics["overall"] >= ADVANCE_THRESHOLD
508
- old_level, old_part = state["level"], state["part"]
509
- complete = False
510
- if passed:
511
- level_index, part_index = LEVELS.index(old_level), PARTS.index(old_part)
512
- if part_index + 1 < len(PARTS):
513
- state["part"] = PARTS[part_index + 1]
514
- state["passage"] = generate_passage(state["level"], state["part"])
515
- message = f"Passed! Moving to {state['part']} at {state['level'].replace('_', ' ')} level."
516
- elif level_index + 1 < len(LEVELS):
517
- state["level"] = LEVELS[level_index + 1]
518
- state["part"] = PARTS[0]
519
- state["passage"] = generate_passage(state["level"], state["part"])
520
- message = f"Level complete! Moving to {state['level'].replace('_', ' ')}."
521
- else:
522
- state["ended"] = True
523
- complete = True
524
- message = "Congratulations! You completed all five levels."
525
- else:
526
- # Keep the same generated passage so the learner can retry the exact same phase.
527
- message = f"Score below {ADVANCE_THRESHOLD:.0f}%. Try the same passage again."
528
- return {
529
- "session_id": session_id, "level": old_level, "part": old_part,
530
- "transcript": transcript, "metrics": metrics, "wer": wer,
531
- "events": events, "pronunciation_words": pronunciation_words,
532
- "passed": passed, "threshold": ADVANCE_THRESHOLD, "message": message,
533
- "next_level": state["level"], "next_part": state["part"],
534
- "next_passage": state["passage"] if not complete else None,
535
- "completed": complete, "ended": state["ended"],
536
- }
537
-
538
- @app.get("/api/pronounce/{word}")
539
- def pronounce(word: str):
540
- word = re.sub(r"[^A-Za-z'-]", "", word)[:60]
541
- if not word:
542
- raise HTTPException(status_code=400, detail="Provide a word to pronounce.")
543
- buf = io.BytesIO()
544
- try:
545
- gTTS(text=word, lang="en", slow=True).write_to_fp(buf)
546
- except Exception as exc:
547
- raise HTTPException(status_code=502, detail=f"Text-to-speech could not generate audio: {str(exc)[:200]}")
548
- buf.seek(0)
549
- return StreamingResponse(buf, media_type="audio/mpeg", headers={"Cache-Control": "no-store"})
550
-
551
- @app.post("/api/session/{session_id}/gesture")
552
- async def gesture(session_id: str, image: UploadFile = File(...)):
553
- state = sessions.get(session_id)
554
- if not state:
555
- raise HTTPException(status_code=404, detail="Session not found. Start a new session.")
556
- raw = await image.read()
557
- gesture_name = detect_gesture_from_bytes(raw)
558
- action = "none"
559
- if gesture_name == "HOME":
560
- state["level"], state["part"] = LEVELS[0], PARTS[0]
561
- state["passage"] = generate_passage(state["level"], state["part"])
562
- state["ended"] = False
563
- action = "home"
564
- elif gesture_name == "PAUSE":
565
- state["ended"] = True
566
- action = "pause"
567
- elif gesture_name.startswith("LEVEL_"):
568
- level_index = int(gesture_name.split("_")[1]) - 1
569
- state["level"], state["part"] = LEVELS[level_index], PARTS[0]
570
- state["passage"] = generate_passage(state["level"], state["part"])
571
- state["ended"] = False
572
- action = "jump_level"
573
- return {"gesture": gesture_name, "action": action, "level": state["level"], "part": state["part"], "passage": state["passage"] if not state["ended"] else None, "ended": state["ended"]}
574
-
575
- @app.post("/api/session/{session_id}/finish")
576
- def finish_session(session_id: str):
577
- state = sessions.get(session_id)
578
- if not state:
579
- raise HTTPException(status_code=404, detail="Session not found. Start a new session.")
580
- state["ended"] = True
581
- history = state["history"]
582
- formats = ["sentence", "paragraph", "story"]
583
- matrix = {actual: {predicted: 0 for predicted in formats} for actual in formats}
584
- for record in generation_records:
585
- actual = record.get("requested_format")
586
- predicted = record.get("observed_format")
587
- if actual in matrix and predicted in matrix[actual]:
588
- matrix[actual][predicted] += 1
589
- format_matches = sum(matrix[f][f] for f in formats)
590
- generation_summary = {
591
- "passages_generated": len(generation_records),
592
- "format_confusion_matrix_rows_requested_columns_detected": matrix,
593
- "heuristic_format_match_rate_percent": round(100 * format_matches / len(generation_records), 2) if generation_records else 0,
594
- "fallback_rate_percent": round(100 * sum(bool(x["used_fallback"]) for x in generation_records) / len(generation_records), 2) if generation_records else 0,
595
- "length_compliance_percent": round(100 * sum(bool(x["length_compliant"]) for x in generation_records) / len(generation_records), 2) if generation_records else 0,
596
- "structure_compliance_percent": round(100 * sum(bool(x["structure_compliant"]) for x in generation_records) / len(generation_records), 2) if generation_records else 0,
597
- "mean_generation_seconds": round(float(np.mean([x["generation_seconds"] for x in generation_records])), 2) if generation_records else 0,
598
- "records": generation_records[-100:],
599
- "note": "Format detection is heuristic; these automatic checks do not establish grammar, coherence, factuality, or true reading-level suitability."
600
- }
601
- return {
602
- "session_id": session_id, "ended": True, "level": state["level"], "part": state["part"],
603
- "attempts": len(history),
604
- "average_word_accuracy": round(float(np.mean([h["metrics"]["word_accuracy"] for h in history])), 2) if history else 0,
605
- "average_overall": round(float(np.mean([h["metrics"]["overall"] for h in history])), 2) if history else 0,
606
- "average_wer": round(float(np.mean([h["wer"] for h in history])), 2) if history else 0,
607
- "history": history,
608
- "text_generation": generation_summary,
609
- }
610
-
611
- # Run with: uvicorn rise_colab_api:app --host 0.0.0.0 --port 8000
 
1
  """RISE reading-assistant API for a Google Colab backend.
2
 
3
+ Uses RISE's own trained encoder-decoder Transformer, Whisper Small,
4
  gTTS pronunciation audio, and MediaPipe Hand Landmarker.
5
  No notebook-only output.eval_js or IPython.display calls are used in API routes.
6
  """
 
29
  from fastapi.responses import StreamingResponse
30
  from pydantic import BaseModel
31
  from gtts import gTTS
32
+ from transformers import pipeline as hf_pipeline
33
+ from rise_custom_transformer import load_custom_text_generator
 
 
 
 
34
 
35
  # --------------------------- Curriculum/configuration -----------------------
36
  ADVANCE_THRESHOLD = 80.0
 
79
 
80
  # --------------------------- Model globals ----------------------------------
81
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
82
+ GENAI_MODEL_NAME = "RISE Custom Encoder-Decoder Transformer"
83
+ custom_text_generator = None
 
 
 
84
  speech_to_text = None
85
  _hand_landmarker = None
86
  model_status: dict[str, Any] = {"whisper": False, "text_generation": False, "gesture": False, "text_model": None, "errors": []}
 
93
  "we're": "we are", "they're": "they are", "you're": "you are",
94
  "isn't": "is not", "aren't": "are not", "wasn't": "was not", "weren't": "were not",
95
  "don't": "do not", "doesn't": "does not", "didn't": "did not", "can't": "cannot",
96
+ "couldn't": "could not", "won't": "will not", "wouldn't": "would not", "there's": "there is",