Merge pull request #3 from francisco-magana/hygienepile-0k/nah-8-grade_answer-strict-json-grading-parserrepair-retry

learning_engine.py

@@ -53,23 +53,41 @@ def grade_answer(card: Card, user_answer: str) -> GradeResult:
 
     messages = [
         {"role": "system", "content":
-         "You grade a student's answer. Return ONLY JSON with keys: "
-         "score (0-5 int), explanation (string), missed_concept (string)."},
+         "You grade a student's answer against a reference answer. "
+         "Return ONLY a JSON object with keys: "
+         "score (integer 0-5), explanation (string for the student), "
+         "missed_concept (short string naming what they got wrong, or \"\")."},
         {"role": "user", "content":
          f"Question: {card['question']}\nReference answer: {card['answer']}\n"
          f"Student answer: {user_answer}\nGrade it."},
     ]
-    data = llm.extract_json(llm.chat(messages, max_tokens=256))
-    if not isinstance(data, dict) or "score" not in data:
-        return new_grade(2, "Couldn't grade automatically — review the reference "
-                            f"answer: {card['answer']}", card["topic"])
+    # Parser + one repair retry; safe default if the model never returns JSON.
+    data = llm.chat_json(messages, max_tokens=256)
+    if not _valid_grade(data):
+        return new_grade(
+            2,
+            "Couldn't grade automatically — compare your answer to the "
+            f"reference: {card['answer']}",
+            card["topic"],
+        )
     return new_grade(
-        int(data.get("score", 0)),
-        str(data.get("explanation", "")).strip(),
-        str(data.get("missed_concept", card["topic"])).strip(),
+        int(data["score"]),
+        str(data.get("explanation", "")).strip()
+        or f"Reference answer: {card['answer']}",
+        str(data.get("missed_concept") or card["topic"]).strip(),
     )
 
 
+def _valid_grade(data) -> bool:
+    """A grade is usable only if it carries a numeric, in-range score."""
+    if not isinstance(data, dict) or "score" not in data:
+        return False
+    try:
+        return 0 <= int(data["score"]) <= 5
+    except (TypeError, ValueError):
+        return False
+
+
 # ---- Adaptation: SM-2-lite -------------------------------------------------
 
 def apply_result(session: Session, card: Card, grade: GradeResult) -> Session:

llm.py

@@ -89,6 +89,32 @@ def extract_json(text: str):
     return None
 
 
+def chat_json(messages: list[dict], max_tokens: int = 256, retries: int = 1):
+    """
+    Call the model and parse its reply as JSON, with up to `retries` repair
+    passes. Model output is never trusted: if the first reply isn't valid JSON
+    we feed it back with a terse "return ONLY valid JSON" nudge and try again.
+
+    Returns the parsed object/array, or None if every attempt fails (callers
+    must handle None with a safe default — never crash the study loop).
+    """
+    convo = list(messages)
+    for attempt in range(retries + 1):
+        reply = chat(convo, max_tokens=max_tokens)
+        data = extract_json(reply)
+        if data is not None:
+            return data
+        if attempt < retries:
+            # Repair pass: show the model its bad reply and demand clean JSON.
+            convo = messages + [
+                {"role": "assistant", "content": reply},
+                {"role": "user", "content":
+                 "That was not valid JSON. Reply again with ONLY the JSON "
+                 "value — no prose, no markdown fences, no trailing text."},
+            ]
+    return None
+
+
 # ---- Stub replies so the app runs with no model ----------------------------
 
 def _stub_reply(messages: list[dict]) -> str:

test_grade_answer.py

@@ -0,0 +1,89 @@
+"""
+NAH-8 — grade_answer strict-JSON grading + parser/repair retry.
+
+Runs with the real (non-stub) code path but with llm.chat monkeypatched, so no
+model/GPU is needed. Verifies: clean JSON parses, a bad first reply triggers one
+repair retry, and a never-valid model gives a safe default instead of crashing.
+
+    RECALL_STUB=0 python3 test_grade_answer.py
+"""
+import os
+
+os.environ["RECALL_STUB"] = "0"  # exercise the real model path, not the heuristic
+
+import llm
+import learning_engine as le
+from schema import new_card
+
+
+def _card():
+    return new_card(
+        question="What does mitochondria do?",
+        answer="It produces ATP, the cell's energy.",
+        topic="Cell Biology",
+    )
+
+
+def _fake_chat(replies):
+    """Return a chat() that yields the given replies in order."""
+    calls = {"n": 0}
+
+    def chat(messages, max_tokens=512):
+        i = min(calls["n"], len(replies) - 1)
+        calls["n"] += 1
+        return replies[i]
+
+    return chat, calls
+
+
+def test_clean_json_first_try():
+    llm.chat, calls = _fake_chat([
+        '{"score": 5, "explanation": "Spot on.", "missed_concept": ""}'
+    ])
+    g = le.grade_answer(_card(), "It makes ATP energy")
+    assert g["score"] == 5 and g["correct"] is True
+    assert g["explanation"] == "Spot on."
+    assert calls["n"] == 1, "should not retry when first reply is valid"
+    print("ok  clean JSON on first try")
+
+
+def test_repair_retry_recovers():
+    # First reply is junk; repair pass returns valid JSON.
+    llm.chat, calls = _fake_chat([
+        "Sure! The student did okay I think, maybe a 2 or 3.",
+        '```json\n{"score": 2, "explanation": "Missed the ATP detail.", '
+        '"missed_concept": "ATP production"}\n```',
+    ])
+    g = le.grade_answer(_card(), "it is in the cell")
+    assert g["score"] == 2 and g["correct"] is False
+    assert g["missed_concept"] == "ATP production"
+    assert calls["n"] == 2, "should retry exactly once to repair bad JSON"
+    print("ok  repair retry recovers bad first reply")
+
+
+def test_safe_default_when_never_valid():
+    llm.chat, calls = _fake_chat(["no json here", "still no json at all"])
+    g = le.grade_answer(_card(), "dunno")
+    assert g["score"] == 2  # neutral safe default
+    assert "reference" in g["explanation"].lower()
+    assert calls["n"] == 2, "tries once + one repair, then gives up"
+    print("ok  safe default when model never returns JSON")
+
+
+def test_out_of_range_score_rejected():
+    # Score outside 0-5 must be treated as unusable, not clamped silently.
+    llm.chat, calls = _fake_chat([
+        '{"score": 99, "explanation": "x"}',
+        'also not valid json',
+    ])
+    g = le.grade_answer(_card(), "whatever")
+    assert g["score"] == 2, "out-of-range score should fall through to default"
+    print("ok  out-of-range score rejected -> safe default")
+
+
+if __name__ == "__main__":
+    test_clean_json_first_try()
+    test_repair_retry_recovers()
+    test_safe_default_when_never_valid()
+    test_out_of_range_score_rejected()
+    print("\nAll NAH-8 grade_answer tests passed.")