sync: src/nodes/socratic_generator.py

src/nodes/socratic_generator.py

@@ -8,6 +8,8 @@ Only visible_response.socratic_question + encouragement reach the student.
 from __future__ import annotations
 
 import os
+import queue
+import threading
 from typing import Literal, Optional
 
 import yaml
@@ -16,6 +18,38 @@ from pydantic import BaseModel
 
 from src.state import TutoringState, Phase
 
+# ── Per-session streaming token queues ───────────────────────────────────────
+# api.py registers a queue before calling graph.invoke; socratic_generator
+# puts tokens in it; api.py drains the queue and yields SSE events.
+# Sentinel None marks end-of-stream.
+
+_token_queues: dict[str, queue.Queue] = {}
+_token_queues_lock = threading.Lock()
+
+
+def register_token_queue(session_id: str) -> queue.Queue:
+    q: queue.Queue = queue.Queue()
+    with _token_queues_lock:
+        _token_queues[session_id] = q
+    return q
+
+
+def unregister_token_queue(session_id: str) -> None:
+    with _token_queues_lock:
+        _token_queues.pop(session_id, None)
+
+
+def _get_token_queue(session_id: str) -> Optional[queue.Queue]:
+    with _token_queues_lock:
+        return _token_queues.get(session_id)
+
+
+def push_phase_event(session_id: str, from_phase: str, to_phase: str) -> None:
+    """Push a phase-change marker into the token queue before any tokens stream."""
+    q = _get_token_queue(session_id)
+    if q is not None:
+        q.put({"_phase_change": True, "from": from_phase, "to": to_phase})
+
 with open("config.yaml") as f:
     _cfg = yaml.safe_load(f)
 
@@ -25,9 +59,9 @@ with open("config.yaml") as f:
 class InternalAnalysis(BaseModel):
     """Stripped by app layer — never shown to student."""
     correct_answer: str
-    student_misconception: str
-    planned_hint_sequence: list[str]
-    relevant_textbook_section: str
+    student_misconception: Optional[str] = ""
+    planned_hint_sequence: list[str] = []
+    relevant_textbook_section: str = ""
 
 
 class VisibleResponse(BaseModel):
@@ -148,16 +182,17 @@ Do NOT add any transition like "let's get started" — the first question will a
 Do NOT ask any anatomy question. Do NOT end with "?". No more than 2 sentences."""
 
 _RAPPORT_SYSTEM = """\
-You are UnMask, a friendly Socratic tutor helping OT students prepare for the NBCOT exam.
+You are UnMask, a Socratic tutor helping OT students prepare for the NBCOT exam.
 You are running a short diagnostic. The next diagnostic question will be shown automatically — \
 you MUST NOT ask any question yourself. Do NOT end your response with "?".
-React to the student's answer in 1-2 sentences ONLY: briefly acknowledge correct/incorrect \
-(without revealing the full answer) and offer brief encouragement. Then stop completely.
-Examples of good responses:
-- "Exactly right — C5 to T1 are the five roots. Nice start!"
-- "Close! You've got the right range in mind. Keep going."
-- "Not quite, but good attempt — we'll revisit that."
-Do not ask follow-up questions. Do not say 'which topic' or 'what would you like to explore'."""
+React to the student's answer in 1 sentence ONLY. Then stop completely.
+
+TONE RULES (apply strictly):
+- If student says "idk", "don't know", "no idea", "not sure", or any phrase under 8 words with no anatomy → respond ONLY with a neutral one-liner e.g. "That one's tricky — we'll build it up." NEVER say "no worries", "that's okay", "don't stress", or anything that sounds like consolation.
+- If correct → brief specific praise e.g. "Exactly — C5 to T1 are the five roots."
+- If partially correct or wrong → neutral acknowledgment e.g. "Not quite — we'll come back to that."
+Do not ask follow-up questions. Do not say 'which topic' or 'what would you like to explore'.
+Do not give study tips or motivational speeches."""
 
 _TUTORING_SYSTEM = """\
 You are UnMask, a warm and encouraging Socratic tutor for OT anatomy (NBCOT prep).
@@ -181,7 +216,7 @@ TURN: {turn} | CONSECUTIVE INCORRECT: {consecutive_incorrect}
 STUDY FOCUS: {study_focus} — ALL questions MUST stay within this topic area. Do not stray to unrelated anatomy concepts. If study_focus starts with "topic:", focus exclusively on that topic's clinical syndromes, mechanisms, and signs. | LEARNING MODE: {learning_mode} — if "visual", use spatial anatomical descriptions and reference diagram layouts; if "text", use clear prose explanations.
 
 TONE GUIDE — encouragement must be exactly ONE sentence, original, not canned:
-- If the student's message is "idk", "don't know", "no idea", "not sure", or any short non-answer (< 10 words with no anatomy content) → neutral pivot ONLY e.g. "No worries, let's build it up together." Do NOT say "you're on the right track", "great job", or any positive feedback regardless of consecutive_incorrect.
+- If the student's message is "idk", "don't know", "no idea", "not sure", "oh", "ok", "hm", or any short non-answer (< 10 words with no anatomy content) → neutral pivot ONLY. VARY the phrasing every turn — do NOT repeat "No worries, let's build it up together." Use alternatives like: "That one catches people — let's approach it differently." / "Fair enough — let's try a clinical angle." / "Totally understandable — here's a starting point." / "Let's back up a bit." Never repeat the same opener twice in a session.
 - consecutive_incorrect = 0 AND student gave a substantive correct answer → specific praise referencing what they got right e.g. "Exactly — C5 to T1 are the five roots."
 - consecutive_incorrect = 1 → warm redirect e.g. "Not quite, but think about the movement involved."
 - consecutive_incorrect = 2 → empathetic e.g. "This one trips a lot of people — let's try a fresh angle."
@@ -189,7 +224,12 @@ TONE GUIDE — encouragement must be exactly ONE sentence, original, not canned:
 CRITICAL: Write ONE sentence only for encouragement. No joining two phrases with a dash or period.
 NEVER say "great job" / "well done" / "you're doing great" / "you've got X right" when the student expressed uncertainty or said they don't know.
 Evaluate ONLY the CURRENT student message — ignore how well they answered in previous turns.
-ANTI-REPETITION: The conversation history above contains your previous responses. Your new socratic_question MUST be DIFFERENT from your last question — use different wording, a different angle, or a more concrete scaffold. If the student has said "idk"/"no idea" twice in a row, do NOT ask the same question again; instead give a specific leading clue or a simpler sub-question.
+CLASSIFY THE STUDENT'S MESSAGE before responding:
+  A) SUBSTANTIVE ANATOMY ATTEMPT — contains anatomy terms, a diagnosis, a mechanism, a description of function, or a genuine attempt at an answer. → Evaluate it, give honest feedback, ask a follow-up that builds on it.
+  B) ACKNOWLEDGMENT / REACTION — no anatomy content; just a social/emotional reaction (e.g. "oh", "ok", "neat", "interesting", "oh nice", "cool", "got it", "makes sense", "clean intro", "oh nice clean neat", "sure", "yeah", "that makes sense", "oh wow"). These could be 1 word or a short casual phrase. → Do NOT repeat your last question. Advance the scaffold with a completely new angle: different clinical scenario, a simpler sub-question, or a concrete clue.
+  C) UNCERTAINTY — "idk", "no idea", "not sure", "still confused", etc. → Same as B: advance, don't repeat.
+  If in doubt between B and C, treat it as B.
+ANTI-REPETITION: Your socratic_question MUST be DIFFERENT from the last assistant message in the conversation history. If the student gave a B/C response, your question MUST use a different scenario and different wording. NEVER return the previous question verbatim or near-verbatim.
 {revisit_block}"""
 
 _REVEAL_SYSTEM = """\
@@ -200,8 +240,10 @@ CONTEXT (textbook source of truth):
 {context}
 
 WHAT TO DO (in visible_response):
-- encouragement: Acknowledge their effort honestly in ONE natural sentence. If they asked directly, say something like "No problem — let me just walk you through it." NOT fake praise.
-- socratic_question: Give the correct answer clearly in 1-2 sentences (you can name it — they need to know it now). Then briefly explain WHY (mechanism/clinical link). End with ONE simple check question like "Does that click now?" or "Want to try a quick follow-up?"
+- encouragement: ONE natural sentence acknowledging their effort. e.g. "No problem — let me walk you through it." NOT fake praise.
+- socratic_question: DIRECTLY EXPLAIN the answer. Name the nerve/muscle/structure explicitly. Explain the mechanism or clinical link in 2-3 sentences. DO NOT ask a new anatomy or clinical scenario question — this is an explanation, not a quiz. You may end with a soft check like "Does that make sense?" or "Does that click?" but NOT a new clinical scenario or anatomy question.
+
+CRITICAL: The student asked for an explanation. Give it. Do NOT respond with another Socratic question about the concept.
 
 In internal_analysis, still compute the correct answer and misconception as usual.
 STUDY FOCUS: {study_focus} | CONSECUTIVE INCORRECT: {consecutive_incorrect}"""
@@ -219,8 +261,8 @@ CONSECUTIVE INCORRECT / NO-ANSWER: {consecutive_incorrect}
 STUDENT'S CURRENT MESSAGE: "{student_message}"
 
 TONE RULES — apply to encouragement field ONLY (1 sentence max):
-- If the student's current message is "idk", "don't know", "no idea", "not sure", or any short non-answer:
-  → encouragement MUST be a neutral pivot e.g. "No worries — let me give you a hint to work from."
+- If the student's current message is "idk", "don't know", "no idea", "not sure", "oh", "ok", "hm", or any short non-answer:
+  → encouragement MUST be a neutral pivot. VARY phrasing — avoid repeating "No worries" or "let's build it up". Use e.g. "That one catches people.", "Let's try a different angle.", "Fair enough — here's a clue."
   → Do NOT say "you're on the right track", "great job", "you've got it", or any positive feedback.
   → socratic_question MUST provide a concrete leading clue or scaffold, not just repeat the question.
 - consecutive_incorrect = 0 AND substantive correct answer → specific praise referencing what they got right.
@@ -382,8 +424,9 @@ resources: 3-4 specific study resources, mixing:
   - NBCOT-specific practice
     Example: 'NBCOT Prep: clinical scenarios — peripheral nerve injury splinting'
 
-youtube_resources: 2-4 curated YouTube videos, one per WEAKEST topic (mastery < 0.5 from mastery_json above).
-  Prioritize the lowest-mastery concepts — only recommend videos for topics the student actually struggled with.
+youtube_resources: 2-4 curated YouTube videos for the weak topics listed above. ALWAYS include at least 2 videos —
+  even if the session was brief, recommend videos for the topics listed in weak_topics.
+  Prioritize the lowest-mastery concepts.
   Use real videos from reputable anatomy channels: Osmosis, Khan Academy, AnatomyZone, Armando Hasudungan,
   Netter Anatomy, Acland's Video Atlas, Ninja Nerd Science.
   Each resource must include:
@@ -428,67 +471,174 @@ def _generate_session_summary(state: TutoringState) -> tuple[str, SessionSummary
         [(k, v) for k, v in visited_mastery.items() if v < 0.5],
         key=lambda x: x[1]
     )
-    weak_topics_str = ", ".join(f"{k} ({v:.0%})" for k, v in weak_topics) or "none"
+
+    # Fallback: if no mastery data, use the study focus topic so YouTube recs are always generated
+    if not weak_topics:
+        fallback = (state.get("study_focus") or "brachial_plexus").replace("topic:", "").strip()
+        weak_topics = [(fallback, 0.0)]
+        if fallback not in topics_visited:
+            topics_visited.add(fallback)
+
+    weak_topics_str = ", ".join(f"{k} ({v:.0%})" for k, v in weak_topics)
 
     client = _get_client()
+    # Cap mistake_log to avoid prompt bloat on long sessions
+    capped_mistakes = mistake_log[-30:] if len(mistake_log) > 30 else mistake_log
     prompt = _SUMMARY_PROMPT.format(
         mastery_json=json.dumps(visited_mastery, indent=2),
-        mistakes_json=json.dumps(mistake_log, indent=2) if mistake_log else "[]",
+        mistakes_json=json.dumps(capped_mistakes, indent=2) if capped_mistakes else "[]",
         topics_covered=", ".join(topics_visited) or "none",
         weak_topics=weak_topics_str,
         duration_min=elapsed / 60,
         total_turns=turn,
     )
 
-    resp = client.beta.chat.completions.parse(
-        model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
-        messages=[{"role": "user", "content": prompt}],
-        response_format=SessionSummary,
-        temperature=0.3,
+    # Use plain create + system-message JSON instruction.
+    # (beta.parse embeds the full Pydantic schema, exceeding Mercury-2's context limit.)
+    # max_tokens=4096: 16-topic report with flashcards+youtube easily hits 3k tokens.
+    import re as _re
+    _SYSTEM_JSON = (
+        "You are a JSON-only assistant. Respond ONLY with a valid JSON object — "
+        "no markdown fences, no prose before or after. "
+        "Required keys: overall_assessment, topic_reports, mistake_highlights, "
+        "study_recommendations, resources, youtube_resources, diagram_suggestions, "
+        "flashcards, next_session_questions, closing_reflection. "
+        "topic_reports items: {concept, mastery_score, status (mastered|progressing|needs_review), honest_feedback}. "
+        "youtube_resources items: {concept, title, creator, search_query, description}. "
+        "flashcards items: {concept, front, back}."
     )
-    summary: SessionSummary = resp.choices[0].message.parsed
+    import logging as _logging
+    raw_content = ""
+    for _attempt in range(2):
+        try:
+            resp_raw = client.chat.completions.create(
+                model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
+                messages=[
+                    {"role": "system", "content": _SYSTEM_JSON},
+                    {"role": "user", "content": prompt},
+                ],
+                max_tokens=4096,
+                temperature=0.3,
+            )
+            raw_content = (resp_raw.choices[0].message.content or "").strip()
+            if raw_content:
+                break
+            _logging.warning(f"[session_summary] attempt {_attempt+1}: empty response from model")
+        except Exception as _api_err:
+            _logging.warning(f"[session_summary] attempt {_attempt+1}: API error {_api_err!r}")
+
+    # Strip markdown code fences and extract first {...} block
+    raw_content = _re.sub(r"```(?:json)?\s*", "", raw_content).replace("```", "").strip()
+    _brace = raw_content.find("{")
+    _rbrace = raw_content.rfind("}")
+    if _brace != -1 and _rbrace != -1 and _rbrace > _brace:
+        raw_content = raw_content[_brace : _rbrace + 1]
+
+    summary: SessionSummary | None = None
+    try:
+        data = json.loads(raw_content)
+        # Fill in any missing optional fields before validation
+        data.setdefault("topic_reports", [])
+        data.setdefault("mistake_highlights", [])
+        data.setdefault("study_recommendations", [])
+        data.setdefault("resources", [])
+        data.setdefault("youtube_resources", [])
+        data.setdefault("diagram_suggestions", [])
+        data.setdefault("flashcards", [])
+        data.setdefault("next_session_questions", [])
+        data.setdefault("closing_reflection", "Keep reviewing!")
+
+        # Normalize list[str] fields — model often returns list[dict] for these
+        def _coerce_str_list(items: list, *keys: str) -> list[str]:
+            result = []
+            for item in items:
+                if isinstance(item, str):
+                    result.append(item)
+                elif isinstance(item, dict):
+                    # Try common keys the model uses, fall back to joined values
+                    text = next((item[k] for k in keys if k in item and isinstance(item[k], str)), None)
+                    if text is None:
+                        text = " — ".join(str(v) for v in item.values() if v)
+                    result.append(text)
+            return result
+
+        data["mistake_highlights"] = _coerce_str_list(
+            data["mistake_highlights"], "description", "text", "misconception", "highlight"
+        )
+        data["resources"] = _coerce_str_list(
+            data["resources"], "title", "description", "text", "resource"
+        )
+        data["study_recommendations"] = _coerce_str_list(
+            data["study_recommendations"], "recommendation", "text", "tip", "description"
+        )
+        data["diagram_suggestions"] = _coerce_str_list(
+            data["diagram_suggestions"], "description", "text", "suggestion", "diagram"
+        )
+        data["next_session_questions"] = _coerce_str_list(
+            data["next_session_questions"], "question", "text", "description"
+        )
+
+        # Tolerate extra keys
+        summary = SessionSummary.model_validate(data)
+    except Exception as _parse_err:
+        _logging.warning(f"[session_summary] JSON parse failed: {_parse_err!r} | raw[:400]={raw_content[:400]!r}")
+
+    if summary is None:
+        fallback = "## 📋 Session Report\n\nSession complete. Great work today — review your weak topics before next time."
+        return fallback, SessionSummary(
+            overall_assessment="Session complete.",
+            topic_reports=[], mistake_highlights=[], study_recommendations=[],
+            resources=[], youtube_resources=[], diagram_suggestions=[],
+            flashcards=[], next_session_questions=[], closing_reflection="Keep reviewing!",
+        )
 
     # ── Format as readable markdown ────────────────────────────────────────
     lines = ["## 📋 Session Report\n"]
-    lines.append(f"{summary.overall_assessment}\n")
+    lines.append(f"{summary.overall_assessment or ''}\n")
 
     # Per-topic report card
-    lines.append("### Topic Breakdown\n")
-    status_icon = {"mastered": "✅", "progressing": "🟡", "needs_review": "❌"}
-    for tr in summary.topic_reports:
-        icon = status_icon.get(tr.status, "⬜")
-        concept_readable = tr.concept.replace("_", " ").replace(".", " › ")
-        lines.append(
-            f"{icon} **{concept_readable}** — mastery {tr.mastery_score:.0%}\n"
-            f"> {tr.honest_feedback}\n"
-        )
+    topic_reports = summary.topic_reports or []
+    if topic_reports:
+        lines.append("### Topic Breakdown\n")
+        status_icon = {"mastered": "✅", "progressing": "🟡", "needs_review": "❌"}
+        for tr in topic_reports:
+            icon = status_icon.get(tr.status, "⬜")
+            concept_readable = tr.concept.replace("_", " ").replace(".", " › ")
+            lines.append(
+                f"{icon} **{concept_readable}** — mastery {tr.mastery_score:.0%}\n"
+                f"> {tr.honest_feedback}\n"
+            )
 
     # Mistake highlights
-    if summary.mistake_highlights:
-        lines.append("### ⚠️ Misconceptions to Address\n")
-        for m in summary.mistake_highlights:
-            lines.append(f"- {m}")
+    for m in (summary.mistake_highlights or []):
+        if not lines[-1].startswith("### ⚠️"):
+            lines.append("### ⚠️ Misconceptions to Address\n")
+        lines.append(f"- {m}")
+    if any("Misconceptions" in l for l in lines):
         lines.append("")
 
     # Study recommendations
-    if summary.study_recommendations:
+    study_recs = summary.study_recommendations or []
+    if study_recs:
         lines.append("### 📚 Study Recommendations\n")
-        for tip in summary.study_recommendations:
+        for tip in study_recs:
             lines.append(f"- {tip}")
         lines.append("")
 
     # Resources
-    if summary.resources:
+    resources = summary.resources or []
+    if resources:
         lines.append("### 📖 Study Resources\n")
-        for r in summary.resources:
+        for r in resources:
             lines.append(f"- {r}")
         lines.append("")
 
     # YouTube recommendations
-    if summary.youtube_resources:
+    youtube = summary.youtube_resources or []
+    if youtube:
         lines.append("### 🎬 Recommended Videos\n")
-        for yt in summary.youtube_resources:
-            search_url = "https://www.youtube.com/results?search_query=" + yt.search_query.replace(" ", "+")
+        for yt in youtube:
+            search_url = "https://www.youtube.com/results?search_query=" + (yt.search_query or "").replace(" ", "+")
             lines.append(
                 f"- **{yt.title}** — *{yt.creator}*\n"
                 f"  {yt.description}\n"
@@ -496,18 +646,17 @@ def _generate_session_summary(state: TutoringState) -> tuple[str, SessionSummary
             )
         lines.append("")
 
-    # Flashcards and diagrams are sent as separate SSE events by api.py
-    # (_send_followup_resources) — omit them here to avoid duplication.
-
     # Next session questions
-    if summary.next_session_questions:
+    next_qs = summary.next_session_questions or []
+    if next_qs:
         lines.append("### 🔁 Practice Questions for Next Session\n")
-        for i, q in enumerate(summary.next_session_questions, 1):
+        for i, q in enumerate(next_qs, 1):
             lines.append(f"**{i}.** {q}")
         lines.append("")
 
     # Closing reflection
-    lines.append(f"---\n**Before next session:** {summary.closing_reflection}")
+    if summary.closing_reflection:
+        lines.append(f"---\n**Before next session:** {summary.closing_reflection}")
 
     return "\n".join(lines), summary
 
@@ -529,23 +678,30 @@ def _generate_assessment_feedback(state: TutoringState, scenario: str, student_r
     """Generate explicit structured feedback on a student's assessment answer."""
     import json
     chunks = state.get("retrieved_chunks", [])
-    context_text = "\n\n".join(
+    _raw_ctx2 = "\n\n".join(
         f"[{c.get('chunk_type','ctx').upper()}] {c['text']}" for c in chunks
     ) or "(no context)"
+    context_text = _raw_ctx2[:8000] + ("\n...[truncated]" if len(_raw_ctx2) > 8000 else "")
 
+    import re as _re2, json as _json2
     client = _get_client()
     prompt = _ASSESSMENT_FEEDBACK_SYSTEM.format(
         scenario=scenario,
         student_response=student_response,
         context=context_text,
     )
-    resp = client.beta.chat.completions.parse(
+    resp_fb = client.chat.completions.create(
         model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
-        messages=[{"role": "user", "content": prompt}],
-        response_format=AssessmentFeedback,
+        messages=[
+            {"role": "system", "content": "You are a JSON-only assistant. Respond ONLY with a valid JSON object. No markdown, no prose."},
+            {"role": "user", "content": prompt},
+        ],
+        max_tokens=600,
         temperature=0.3,
+        response_format={"type": "json_object"},
     )
-    fb: AssessmentFeedback = resp.choices[0].message.parsed
+    raw_fb = _re2.sub(r"^```(?:json)?\s*|\s*```$", "", (resp_fb.choices[0].message.content or "").strip(), flags=_re2.DOTALL).strip()
+    fb: AssessmentFeedback = AssessmentFeedback(**_json2.loads(raw_fb))
 
     score_icon = {"excellent": "✅", "good": "🟢", "partial": "🟡", "needs_work": "❌"}.get(fb.score, "⬜")
     lines = [
@@ -572,13 +728,15 @@ def socratic_generator(state: TutoringState) -> dict:
     history = state.get("conversation_history", [])
     mastery = state.get("mastery_scores", {})
     topic = state.get("current_topic", "")
+    _session_id = state.get("session_id", "")
 
     _max_turns = _cfg["llm"].get("max_history_turns", 8)
 
-    context_text = "\n\n".join(
+    _raw_ctx = "\n\n".join(
         f"[{c.get('chunk_type','context').upper()}] {c['text']}"
         for c in chunks
     ) or "(No context retrieved)"
+    context_text = _raw_ctx[:12000] + ("\n...[truncated]" if len(_raw_ctx) > 12000 else "")
 
     recent_history = "\n".join(
         f"{m['role'].capitalize()}: {m['content']}" for m in history[-_max_turns:]
@@ -654,6 +812,9 @@ def socratic_generator(state: TutoringState) -> dict:
         "give me the answer", "i give up", "i have no idea", "no idea whatsoever",
         "i don't know at all", "i dont know at all", "i'm clueless", "im clueless",
         "i don't have time", "dont have time",
+        "explain this to me", "can you explain", "explain this", "explain it to me",
+        "explain in detail", "explain to me", "help me understand", "break it down",
+        "walk me through", "just explain", "please explain",
     )
     consecutive_incorrect = state.get("consecutive_incorrect", 0)
     _student_msg_lower = (state.get("student_message") or "").lower()
@@ -748,21 +909,90 @@ def socratic_generator(state: TutoringState) -> dict:
     internal_analysis = None
     response_text = ""
 
-    for attempt in range(2):
-        for api_attempt in range(3):
+    def _parse_attempt(msgs: list, temp: float) -> SocraticOutput:
+        """Single API call with JSON prompt; streams tokens to queue if registered."""
+        import re as _re, json as _json
+
+        json_instruction = (
+            "\n\nRespond ONLY with a JSON object matching this schema (no markdown, no prose):\n"
+            '{"internal_analysis": {"correct_answer": "...", "student_misconception": "...", '
+            '"planned_hint_sequence": [], "relevant_textbook_section": "..."}, '
+            '"visible_response": {"encouragement": "...", "socratic_question": "...?"}}'
+        )
+        augmented = list(msgs)
+        if augmented and augmented[0]["role"] == "system":
+            augmented[0] = {**augmented[0], "content": augmented[0]["content"] + json_instruction}
+
+        token_q = _get_token_queue(_session_id)
+
+        if token_q is not None:
+            # Streaming path: collect full response, then push visible text as one token.
+            # Mercury 2 outputs ~1000 JSON tokens; character-level streaming of JSON keys
+            # leaks syntax to the frontend. Instead we stream=True for latency benefit
+            # (Mercury 2 starts generating immediately) and emit the visible question once done.
+            stream_resp = client.chat.completions.create(
+                model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
+                temperature=temp,
+                messages=augmented,
+                max_tokens=1000,
+                stream=True,
+            )
+            raw_parts: list[str] = []
+            for chunk in stream_resp:
+                delta = chunk.choices[0].delta.content if chunk.choices else None
+                if delta:
+                    raw_parts.append(delta)
+            raw = "".join(raw_parts).strip()
+            # Extract visible text from JSON, emit as single token
             try:
-                resp = client.beta.chat.completions.parse(
-                    model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
-                    temperature=_cfg["llm"]["temperature"] if attempt == 0 else 0,
-                    messages=messages,
-                    response_format=SocraticOutput,
-                )
-                break  # success
-            except Exception as e:
-                if api_attempt == 2:
-                    raise
-                time.sleep(2 ** api_attempt)  # exponential backoff: 1s, 2s, 4s
-        output: SocraticOutput = resp.choices[0].message.parsed
+                clean = _re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=_re.DOTALL).strip()
+                data = _json.loads(clean)
+                vr = data.get("visible_response", {})
+                enc = vr.get("encouragement", "")
+                sq = vr.get("socratic_question", "")
+                visible_text = f"{enc} {sq}".strip() if enc else sq
+                if visible_text:
+                    token_q.put(visible_text)
+            except Exception:
+                # Fallback: extract question via regex
+                m = _re.search(r'"socratic_question"\s*:\s*"((?:[^"\\]|\\.)*)"', raw)
+                if m:
+                    token_q.put(m.group(1).replace('\\"', '"').replace('\\n', ' '))
+            token_q.put(None)  # end sentinel
+        else:
+            resp = client.chat.completions.create(
+                model=os.getenv("OPENAI_MODEL", _cfg["llm"]["model"]),
+                temperature=temp,
+                messages=augmented,
+                max_tokens=1000,
+            )
+            raw = (resp.choices[0].message.content or "").strip()
+
+        # Try JSON parse → Pydantic
+        try:
+            clean = _re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=_re.DOTALL).strip()
+            data = _json.loads(clean)
+            return SocraticOutput(**data)
+        except Exception as _e:
+            import logging as _logging
+            _logging.warning(f"[socratic_generator] JSON parse failed (attempt): {_e!r} | raw[:300]={raw[:300]!r}")
+
+        # Fallback: regex-extract socratic_question, never return raw JSON
+        m_fallback = _re.search(r'"socratic_question"\s*:\s*"((?:[^"\\]|\\.)*)"', raw)
+        if m_fallback:
+            question = m_fallback.group(1).replace('\\"', '"').replace('\\n', ' ')
+        else:
+            question = "Let's try a different angle — what movement or sensation would be affected if this structure were damaged?"
+        return SocraticOutput(
+            internal_analysis=InternalAnalysis(
+                correct_answer="", student_misconception="",
+                planned_hint_sequence=[], relevant_textbook_section="",
+            ),
+            visible_response=VisibleResponse(encouragement="", socratic_question=question),
+        )
+
+    for attempt in range(2):
+        output: SocraticOutput = _parse_attempt(messages, _cfg["llm"]["temperature"] if attempt == 0 else 0)
         visible = output.visible_response
         candidate = visible.socratic_question
         if visible.encouragement:
@@ -790,9 +1020,9 @@ def socratic_generator(state: TutoringState) -> dict:
             cand_words = set(visible.socratic_question.lower().split())
             prev_words = set(last_assistant_q.lower().split())
             overlap = len(cand_words & prev_words) / max(len(cand_words), 1)
-            if overlap > 0.65:
+            if overlap > 0.45:  # stricter — catch close paraphrases too
                 messages = [
-                    {"role": "system", "content": system + "\n\nYou just asked: \"" + last_assistant_q + "\"\nThe student still doesn't know. Do NOT repeat or rephrase that question. Ask a completely different, simpler sub-question or give a concrete leading clue instead."},
+                    {"role": "system", "content": system + "\n\nCRITICAL: You JUST asked this question and must NOT repeat it: \"" + last_assistant_q[:200] + "\"\nThe student replied with a short acknowledgment. Move to a DIFFERENT approach: different clinical scenario, simpler sub-question, or direct clue. Even a single shared scenario detail counts as repeating."},
                     *history[-10:],
                     {"role": "user", "content": user_msg},
                 ]