Update app.py

app.py

@@ -825,6 +825,101 @@ def extract_causal_signals(
 
     return signals
 
+# ============================================================
+# TOKEN-LEVEL HALLUCINATION HTML BUILDER
+# ============================================================
+
+def build_highlight_html(token_strings, flagged_set, burst_set, anomaly_scores):
+    """
+    Build an inline HTML token map.
+    Red   = anomaly above threshold (hallucinated)
+    Amber = RSNN burst spike
+    Hover tooltip shows token index and anomaly score.
+    """
+
+    legend = (
+        '<div style="font-family:sans-serif;font-size:12px;'
+        'margin-bottom:8px;display:flex;gap:14px;flex-wrap:wrap;">'
+        '<span>'
+        '<span style="background:#ffd8d7;color:#82071e;border-radius:3px;'
+        'padding:1px 6px;font-weight:700;">H</span>'
+        '&nbsp;Hallucinated (anomaly &gt; threshold)'
+        '</span>'
+        '<span>'
+        '<span style="background:#fff3cd;color:#7d4e00;border-radius:3px;'
+        'padding:1px 6px;font-weight:700;">B</span>'
+        '&nbsp;RSNN burst spike'
+        '</span>'
+        '</div>'
+    )
+
+    parts = []
+
+    for i, raw in enumerate(token_strings):
+
+        # Humanise the token string
+        display = raw
+        for src, dst in [
+            ('\u0120', ' '),   # Ġ  → space  (GPT-2/Llama BPE)
+            ('\u2581', ' '),   # ▁  → space  (SentencePiece)
+            ('\u010a', '\n'),  # Ċ  → newline
+            ('<0x0A>', '\n'),  # literal newline token
+        ]:
+            display = display.replace(src, dst)
+
+        # HTML-escape, then convert newlines to <br>
+        display = (
+            display
+            .replace('&', '&amp;')
+            .replace('<', '&lt;')
+            .replace('>', '&gt;')
+            .replace('\n', '<br>')
+        )
+
+        if display.strip() == '':
+            display = '&nbsp;'
+
+        score = (
+            float(anomaly_scores[i])
+            if i < len(anomaly_scores)
+            else 0.0
+        )
+
+        title = f"[{i}] {raw} | anomaly={score:.4f}"
+
+        if i in flagged_set:
+            style = (
+                "background:#ffd8d7;color:#82071e;"
+                "border-radius:3px;padding:1px 3px;cursor:default;"
+            )
+        elif i in burst_set:
+            style = (
+                "background:#fff3cd;color:#7d4e00;"
+                "border-radius:3px;padding:1px 3px;cursor:default;"
+            )
+        else:
+            style = "cursor:default;"
+
+        parts.append(
+            f'<span style="{style}" title="{title}">{display}</span>'
+        )
+
+    text_html = (
+        '<div style="font-family:sans-serif;font-size:14px;'
+        'line-height:1.9;background:#fff;border:1px solid #d0d7de;'
+        'border-radius:8px;padding:12px 16px;word-break:break-word;">'
+        + ''.join(parts)
+        + '</div>'
+    )
+
+    heading = (
+        '<h4 style="font-family:sans-serif;margin-bottom:6px;'
+        'color:#1f2328;">Token-level Hallucination Map</h4>'
+    )
+
+    return heading + legend + text_html
+
+
 # ============================================================
 # CHATBOT + RSNN DETECTION
 # ============================================================
@@ -922,7 +1017,8 @@ def respond(
 
             return (
                 "The model produced "
-                "no scoreable response."
+                "no scoreable response.",
+                "<p><em>No tokens to visualize.</em></p>",
             )
 
 
@@ -1061,6 +1157,25 @@ def respond(
         )
 
 
+        # ----------------------------------------------------
+        # BUILD TOKEN-LEVEL HTML VISUALIZATION
+        # ----------------------------------------------------
+
+        anomaly_list = (
+            anomaly
+            .detach()
+            .cpu()
+            .tolist()
+        )
+
+        hallu_html = build_highlight_html(
+            token_strings,
+            flagged_set=set(flagged_indices),
+            burst_set=set(burst_indices),
+            anomaly_scores=anomaly_list,
+        )
+
+
         flagged_preview = [
 
             f"{i}:{token_strings[i]}"
@@ -1179,7 +1294,8 @@ def respond(
             +
             "\n\n"
             +
-            report
+            report,
+            hallu_html,
         )
 
 
@@ -1201,7 +1317,7 @@ with gr.Blocks() as demo:
         """
 # LatticeSpike Hallucination-Aware Chatbot
 
-**LLM:** Meta Llama 3.2 1B Instruct  
+**LLM:** Meta Llama 3.2 1B Instruct
 **Detector:** LatticeSpike RSNN
 
 The LLM generates an answer, then LatticeSpike analyzes
@@ -1212,6 +1328,16 @@ For this first version, each message is evaluated independently.
     )
 
 
+    hallu_viz = gr.HTML(
+        value=(
+            "<p style='font-family:sans-serif;color:#57606a;font-size:13px;'>"
+            "<em>Submit a question to see the token-level hallucination map.</em>"
+            "</p>"
+        ),
+        label="Token-level Hallucination Visualization",
+    )
+
+
     chatbot = gr.ChatInterface(
 
         fn=respond,
@@ -1242,6 +1368,8 @@ For this first version, each message is evaluated independently.
                 label="Top-p",
             ),
         ],
+
+        additional_outputs=[hallu_viz],
     )