Merge pull request #11 from TheAnkurGoswami/gestures

agent.py

@@ -1,9 +1,11 @@
 import logging
+import textwrap
 import time
 from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
 from langchain_core.output_parsers import StrOutputParser
 from langchain_core.messages import HumanMessage, AIMessage
 from langchain_core.runnables import RunnableLambda
+from gestures import GESTURE_INSTRUCTIONS
 from scenario import get_scenario, DEFAULT_SCENARIO_ID
 from model import get_model
 
@@ -22,12 +24,20 @@ def _log_llm_output(message):
 _logger = RunnableLambda(_log_llm_output)
 
 
-# The system prompt is just the scenario's assembled persona (context.md +
-# character.md), precomputed once at scenario load time. No per-turn emotion
-# variable.
+# The system prompt is the scenario's assembled persona (context.md +
+# character.md), precomputed once at scenario load time, plus the gesture
+# vocabulary every character can draw on (empty when GESTURES_ENABLED is
+# off — see gestures.py). No per-turn emotion variable.
 prompt = ChatPromptTemplate.from_messages(
     [
-        ("system", "{persona}"),
+        (
+            "system",
+            textwrap.dedent(f"""
+                {{persona}}
+
+                {GESTURE_INSTRUCTIONS}
+                """).strip(),
+        ),
         MessagesPlaceholder(variable_name="history"),
         ("human", "{learner_text}"),
     ]

app.py

@@ -1,5 +1,6 @@
 import html
 import os
+import re
 import sys
 import time
 
@@ -42,6 +43,20 @@ from auto import TraineeAgent
 from trainee_archetypes import ARCHETYPE_IDS
 from scenario import get_scenario as _local_scenario, DEFAULT_SCENARIO_ID
 
+# Same "<tag>...</tag>" shape as gestures._GESTURE_TAG_RE, but matched after
+# html.escape(), so the brackets arrive as &lt; / &gt;.
+_ESCAPED_GESTURE_RE = re.compile(r"&lt;(\w+)&gt;(.*?)&lt;/\1&gt;", re.DOTALL)
+
+
+def _render_reply(text: str) -> str:
+    """Escape an AI reply for the chat bubble and show its gesture tags as
+    visible labels. Unescaped, the browser swallows <nodding> etc. as unknown
+    HTML elements and the gestures silently disappear."""
+    return _ESCAPED_GESTURE_RE.sub(
+        r"<span class='gesture-label'>\1</span><span class='gesture-span'>\2</span>",
+        html.escape(text),
+    )
+
 st.set_page_config(
     page_title="ALAS — Manager Coaching Demo",
     page_icon="🎓",
@@ -106,6 +121,19 @@ st.markdown(
         max-width: 80%;
         font-size: 0.95rem;
     }
+    .gesture-label {
+        font-size: 0.7rem;
+        font-style: italic;
+        opacity: 0.75;
+        border: 1px solid currentColor;
+        border-radius: 8px;
+        padding: 0 6px;
+        margin-right: 4px;
+    }
+    .gesture-span {
+        text-decoration: underline dotted;
+        text-underline-offset: 3px;
+    }
     .bubble-student {
         background-color: #E8E0CE;
         color: #1a1a1a;
@@ -561,7 +589,7 @@ def render_messages():
                 unsafe_allow_html=True,
             )
             st.markdown(
-                f"<div class='bubble-manager'>{msg['text']}</div>",
+                f"<div class='bubble-manager'>{_render_reply(msg['text'])}</div>",
                 unsafe_allow_html=True,
             )
 
@@ -635,11 +663,11 @@ def _stream_and_append(trainee_text: str, turn_num: int):
     for token in stream_message(st.session_state.chat_id, trainee_text):
         accumulated += token
         placeholder.markdown(
-            f"<div class='bubble-manager'>{accumulated}▌</div>",
+            f"<div class='bubble-manager'>{_render_reply(accumulated)}▌</div>",
             unsafe_allow_html=True,
         )
     placeholder.markdown(
-        f"<div class='bubble-manager'>{accumulated}</div>",
+        f"<div class='bubble-manager'>{_render_reply(accumulated)}</div>",
         unsafe_allow_html=True,
     )
     st.session_state.messages.append(

assessor.py

@@ -2,6 +2,7 @@ import logging
 
 from langchain_core.output_parsers import JsonOutputParser
 
+from gestures import strip_gestures
 from rubric import get_rubric, Rubric
 from agent import BACKEND
 from model import get_model
@@ -13,10 +14,13 @@ logger = logging.getLogger("alas.assessor")
 
 
 def build_transcript(messages: list[dict], scenario: Scenario) -> str:
+    """Gesture tags are stripped from manager turns: the assessor grades the
+    words spoken, not the avatar's stage directions."""
     lines = []
     for msg in messages:
         role = scenario.learner_name if msg["role"] == "trainee" else scenario.ai_name
-        lines.append(f"{role}: {msg['text']}")
+        text = msg["text"] if msg["role"] == "trainee" else strip_gestures(msg["text"])
+        lines.append(f"{role}: {text}")
     return "\n\n".join(lines)
 
 

gestures.py

@@ -0,0 +1,92 @@
+"""
+Gesture vocabulary for the roleplay avatar.
+
+A gesture is a paired tag — <tag>...</tag> — that the character LLM wraps
+around the words it's saying while performing that gesture, e.g.:
+
+    <nodding>Sure, that makes sense.</nodding> But I still need the numbers.
+
+The wrapped text is still part of the spoken reply; the tag itself is
+metadata for the avatar layer to key its animation off of, not something a
+trainee should ever see literally.
+
+Each tag below names an animation clip the avatar renderer actually has
+("Nodding.mp4", "Distracted.mp4", "LooksAtPhone.mp4", ...). "Idle" is
+deliberately not a gesture here — it's the renderer's default rest state,
+played whenever no tag is active, not something the LLM opts into.
+Transition clips between states (e.g. "IdleToNodding.mp4") are the
+renderer's concern, not the LLM's — it only ever picks a destination state.
+
+Kept in one shared file, not per-scenario, because the tag vocabulary is a
+property of the avatar renderer, not of any one character — every scenario's
+persona references the same GESTURES list via gestures_block().
+"""
+
+import os
+import re
+import textwrap
+from dataclasses import dataclass
+
+# Off switch for the whole gesture vocabulary — set GESTURES_ENABLED=false
+# to have characters reply as plain text, e.g. for a text-only frontend
+# that has no avatar to animate.
+GESTURES_ENABLED = os.getenv("GESTURES_ENABLED", "true").strip().lower() == "true"
+
+# Matches "<tag>...</tag>", tag name captured and reused to find the closing
+# tag, DOTALL so a wrapped span can cross a sentence break.
+_GESTURE_TAG_RE = re.compile(r"<(\w+)>(.*?)</\1>", re.DOTALL)
+
+
+@dataclass(frozen=True)
+class Gesture:
+    tag: str  # token name, used as <tag>...</tag>
+    description: str  # when the character should reach for this gesture
+
+
+GESTURES: list[Gesture] = [
+    Gesture("nodding", "Agreement, acknowledgement, or active listening."),
+    Gesture("distracted", "Trailing off, momentarily elsewhere, not fully present in the conversation."),
+    Gesture("looks_at_phone", "Checking a phone — impatience, a real interruption, or divided attention."),
+]
+
+_GESTURE_TAGS = {g.tag for g in GESTURES}
+
+
+def gestures_block() -> str:
+    """All gestures as one prompt-injectable block, one line per tag."""
+    return "\n".join(f"- <{g.tag}>...</{g.tag}> — {g.description}" for g in GESTURES)
+
+
+def gesture_tags() -> list[str]:
+    return [g.tag for g in GESTURES]
+
+
+# The block a character's system prompt appends to gain gesture control.
+# Empty when GESTURES_ENABLED is off, so callers can splice this in
+# unconditionally rather than checking the flag themselves. Built once at
+# import time — the vocabulary is fixed, not per-scenario or per-turn.
+GESTURE_INSTRUCTIONS = (
+    textwrap.dedent(f"""\
+        ## Gestures
+        You control an avatar that can perform physical gestures while it
+        speaks. Available gestures:
+        {gestures_block()}
+
+        To perform one, wrap the words you're saying while doing it in a tag
+        named after the gesture, e.g. <nodding>Sure, that makes sense.</nodding>
+        A reply may contain several tagged spans, or none — most turns need no
+        gesture at all. Only use a gesture where it would naturally read on a
+        real person's face or body in that moment. Never invent a tag that
+        isn't in the list above, and never mention the tags or describe the
+        gesture in words — the tag itself is the only signal.
+        """)
+    if GESTURES_ENABLED
+    else ""
+)
+
+
+def strip_gestures(text: str) -> str:
+    """Drop gesture tags but keep the words they wrapped. For feeding a
+    character's past reply back into a non-avatar-facing prompt (grading,
+    transcripts) where the tags are just noise."""
+    return _GESTURE_TAG_RE.sub(r"\2", text)

main.py

@@ -19,6 +19,7 @@ import threading
 import uuid
 from fastapi import BackgroundTasks, FastAPI, HTTPException
 from fastapi.responses import StreamingResponse
+from gestures import strip_gestures
 from rubric import get_rubric
 from agent import CharacterAgent
 from assessor import Assessor
@@ -79,10 +80,11 @@ def _get_session(chat_id: str) -> dict:
 def _last_ai_text(messages: list) -> str:
     """The AI turn that preceded (and prompted) the trainee's next message —
     never the AI's reply to that same message, which doesn't exist yet when
-    the trainee is judged."""
+    the trainee is judged. Gesture tags are stripped: this text feeds the
+    router/judge prompts as plain context, not the avatar."""
     for m in reversed(messages):
         if m["role"] == "manager":
-            return m["text"]
+            return strip_gestures(m["text"])
     return ""