Improve perceived and actual response latency

- Reuse a single httpx.AsyncClient across requests (FastAPI lifespan)
instead of opening a fresh connection to api.anthropic.com on every
message, avoiding a repeated TCP+TLS handshake per turn.
- Stream Claude's reply to the browser as plain text (Anthropic SSE
parsed server-side) instead of waiting for the full completion, so
ARI tool replies and partner messages render incrementally.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

app.py

@@ -8,15 +8,31 @@ ARI never becomes the relationship - it augments human-to-human connection.
 Created by Jocelyn Skillman, LMHC
 """
 
+import json
 import os
+from contextlib import asynccontextmanager
+
 import httpx
 from fastapi import FastAPI, HTTPException
 from fastapi.staticfiles import StaticFiles
-from fastapi.responses import FileResponse
+from fastapi.responses import FileResponse, StreamingResponse
 from pydantic import BaseModel
 from typing import List, Optional
 
-app = FastAPI(title="Tend & Send ARI Prototype")
+# Reused across requests so calls to api.anthropic.com share a warm
+# connection pool instead of paying a fresh TCP+TLS handshake every turn.
+http_client: httpx.AsyncClient | None = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    global http_client
+    http_client = httpx.AsyncClient(timeout=60.0)
+    yield
+    await http_client.aclose()
+
+
+app = FastAPI(title="Tend & Send ARI Prototype", lifespan=lifespan)
 
 # Get API key from environment (set as HF Space secret)
 # Check multiple possible secret names for flexibility
@@ -504,13 +520,51 @@ class ChatRequest(BaseModel):
     max_tokens: int = 500
 
 
+# ============================================================================
+# STREAMING HELPER
+# ============================================================================
+
+async def stream_anthropic_text(payload: dict):
+    """POST to the Anthropic Messages API with streaming enabled and yield
+    plain-text chunks as they arrive, so the browser can render the reply
+    incrementally instead of waiting for the full completion."""
+    payload = {**payload, "stream": True}
+    async with http_client.stream(
+        "POST",
+        "https://api.anthropic.com/v1/messages",
+        headers={
+            "Content-Type": "application/json",
+            "x-api-key": ANTHROPIC_API_KEY,
+            "anthropic-version": "2023-06-01"
+        },
+        json=payload
+    ) as response:
+        if response.status_code >= 400:
+            body = await response.aread()
+            raise HTTPException(status_code=response.status_code, detail=body.decode(errors="replace"))
+
+        async for line in response.aiter_lines():
+            if not line.startswith("data: "):
+                continue
+            data = line[len("data: "):]
+            try:
+                event = json.loads(data)
+            except json.JSONDecodeError:
+                continue
+            if event.get("type") == "content_block_delta":
+                delta = event.get("delta", {})
+                text = delta.get("text")
+                if text:
+                    yield text
+
+
 # ============================================================================
 # API ENDPOINTS
 # ============================================================================
 
 @app.post("/api/tool")
 async def use_tool(request: ToolRequest):
-    """Process an ARI tool request"""
+    """Process an ARI tool request, streaming the reply as plain text."""
     if not ANTHROPIC_API_KEY:
         raise HTTPException(status_code=500, detail="API key not configured")
 
@@ -547,66 +601,30 @@ MINIMAL MODE (user wants just the essentials):
         user_message += f"Current stage: {request.stage}\n\n"
     user_message += request.user_input
 
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.post(
-                "https://api.anthropic.com/v1/messages",
-                headers={
-                    "Content-Type": "application/json",
-                    "x-api-key": ANTHROPIC_API_KEY,
-                    "anthropic-version": "2023-06-01"
-                },
-                json={
-                    "model": "claude-sonnet-4-20250514",
-                    "max_tokens": 1000,
-                    "temperature": 0.4,
-                    "system": system_prompt,
-                    "messages": [{"role": "user", "content": user_message}]
-                },
-                timeout=60.0
-            )
-            response.raise_for_status()
-            result = response.json()
-            return {
-                "tool": request.tool,
-                "response": result["content"][0]["text"]
-            }
-        except httpx.HTTPStatusError as e:
-            raise HTTPException(status_code=e.response.status_code, detail=str(e))
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
+    payload = {
+        "model": "claude-sonnet-4-20250514",
+        "max_tokens": 1000,
+        "temperature": 0.4,
+        "system": system_prompt,
+        "messages": [{"role": "user", "content": user_message}]
+    }
+    return StreamingResponse(stream_anthropic_text(payload), media_type="text/plain; charset=utf-8")
 
 
 @app.post("/api/chat")
 async def chat(request: ChatRequest):
-    """General chat endpoint for partner simulation"""
+    """General chat endpoint for partner simulation, streamed as plain text."""
     if not ANTHROPIC_API_KEY:
         raise HTTPException(status_code=500, detail="API key not configured")
 
-    async with httpx.AsyncClient() as client:
-        try:
-            response = await client.post(
-                "https://api.anthropic.com/v1/messages",
-                headers={
-                    "Content-Type": "application/json",
-                    "x-api-key": ANTHROPIC_API_KEY,
-                    "anthropic-version": "2023-06-01"
-                },
-                json={
-                    "model": "claude-sonnet-4-20250514",
-                    "max_tokens": request.max_tokens,
-                    "temperature": 0.5,
-                    "system": request.system,
-                    "messages": [{"role": m.role, "content": m.content} for m in request.messages]
-                },
-                timeout=60.0
-            )
-            response.raise_for_status()
-            return response.json()
-        except httpx.HTTPStatusError as e:
-            raise HTTPException(status_code=e.response.status_code, detail=str(e))
-        except Exception as e:
-            raise HTTPException(status_code=500, detail=str(e))
+    payload = {
+        "model": "claude-sonnet-4-20250514",
+        "max_tokens": request.max_tokens,
+        "temperature": 0.5,
+        "system": request.system,
+        "messages": [{"role": m.role, "content": m.content} for m in request.messages]
+    }
+    return StreamingResponse(stream_anthropic_text(payload), media_type="text/plain; charset=utf-8")
 
 
 @app.get("/api/health")

static/index.html

@@ -1126,6 +1126,35 @@
         let lastPartnerMessage = '';
         let conversationStarted = false;
 
+        // Streams a POST response's body as it arrives, calling onChunk with
+        // each piece of text, and resolves with the full accumulated text.
+        // Falls back to reading the whole body at once on error responses.
+        async function streamFetch(url, body, onChunk) {
+            const response = await fetch(url, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(body)
+            });
+
+            if (!response.ok) {
+                let detail = 'Request failed.';
+                try { detail = (await response.json()).detail || detail; } catch (e) {}
+                throw new Error(detail);
+            }
+
+            const reader = response.body.getReader();
+            const decoder = new TextDecoder();
+            let full = '';
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done) break;
+                const chunk = decoder.decode(value, { stream: true });
+                full += chunk;
+                onChunk(chunk, full);
+            }
+            return full;
+        }
+
         const TOOL_MAP = {
             tend: { name: 'TEND Transform', icon: '✨', api: 'tend' },
             soma: { name: 'Somatic Check-in', icon: '🫁', api: 'somatic_checkin' },
@@ -1573,30 +1602,30 @@ Real grief, real limits. 1-2 sentences.`
             addToolLoading();
             const isVerbose = document.getElementById('toggle-verbose')?.checked || false;
 
+            let textEl = null;
             try {
-                const response = await fetch('/api/tool', {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({
-                        tool: apiTool,
-                        partner_message: lastPartnerMessage,
-                        user_draft: userInput,
-                        user_input: inputText,
-                        verbose: isVerbose
-                    })
+                const full = await streamFetch('/api/tool', {
+                    tool: apiTool,
+                    partner_message: lastPartnerMessage,
+                    user_draft: userInput,
+                    user_input: inputText,
+                    verbose: isVerbose
+                }, (chunk, acc) => {
+                    if (!textEl) {
+                        removeToolLoading();
+                        textEl = addToolMsg('tool', '', TOOL_MAP[tool]?.name);
+                    }
+                    textEl.innerHTML = formatText(acc);
                 });
 
-                const data = await response.json();
-                removeToolLoading();
-
-                if (data.response) {
-                    addToolMsg('tool', data.response, TOOL_MAP[tool]?.name);
-                } else {
+                if (!full) {
+                    removeToolLoading();
                     addToolMsg('tool', 'Could not process. Check API.', 'Error');
                 }
             } catch (error) {
                 removeToolLoading();
-                addToolMsg('tool', 'Connection error.', 'Error');
+                if (textEl) { textEl.innerHTML = formatText('Connection error.'); }
+                else { addToolMsg('tool', 'Connection error.', 'Error'); }
             }
         }
 
@@ -1611,32 +1640,31 @@ Real grief, real limits. 1-2 sentences.`
 
                 addToolLoading();
                 const isVerbose = document.getElementById('toggle-verbose')?.checked || false;
+                let textEl = null;
                 try {
-                    const response = await fetch('/api/tool', {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify({
-                            tool: 'feelings_needs',
-                            partner_message: lastPartnerMessage,
-                            user_draft: '',
-                            user_input: `The person identified these feelings: ${feelingsText}. ${userInput ? 'They added: ' + userInput : ''}
+                    await streamFetch('/api/tool', {
+                        tool: 'feelings_needs',
+                        partner_message: lastPartnerMessage,
+                        user_draft: '',
+                        user_input: `The person identified these feelings: ${feelingsText}. ${userInput ? 'They added: ' + userInput : ''}
 
 Be BRIEF (3-5 sentences max). No headers, no bullet points, no explicit citations.
 
 Validate the feeling. Share one insight about what this feeling tends to signal in relationships or the nervous system. Name what need might be underneath. Keep it warm and conversational - building emotional intelligence, not lecturing.
 
 Draw wisdom from attachment neuroscience, somatic awareness, and relational psychology - but embed it naturally, don't name sources.`,
-                            verbose: isVerbose
-                        })
+                        verbose: isVerbose
+                    }, (chunk, acc) => {
+                        if (!textEl) {
+                            removeToolLoading();
+                            textEl = addToolMsg('tool', '', 'Feelings');
+                        }
+                        textEl.innerHTML = formatText(acc);
                     });
-                    const data = await response.json();
-                    removeToolLoading();
-                    if (data.response) {
-                        addToolMsg('tool', data.response, 'Feelings');
-                    }
                 } catch (e) {
                     removeToolLoading();
-                    addToolMsg('tool', 'Connection error.', 'Error');
+                    if (textEl) { textEl.innerHTML = formatText('Connection error.'); }
+                    else { addToolMsg('tool', 'Connection error.', 'Error'); }
                 }
                 input.value = '';
                 return;
@@ -1649,32 +1677,31 @@ Draw wisdom from attachment neuroscience, somatic awareness, and relational psyc
 
                 addToolLoading();
                 const isVerbose = document.getElementById('toggle-verbose')?.checked || false;
+                let textEl = null;
                 try {
-                    const response = await fetch('/api/tool', {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify({
-                            tool: 'feelings_needs',
-                            partner_message: lastPartnerMessage,
-                            user_draft: '',
-                            user_input: `The person identified these needs: ${needsText}. ${userInput ? 'They added: ' + userInput : ''}
+                    await streamFetch('/api/tool', {
+                        tool: 'feelings_needs',
+                        partner_message: lastPartnerMessage,
+                        user_draft: '',
+                        user_input: `The person identified these needs: ${needsText}. ${userInput ? 'They added: ' + userInput : ''}
 
 Be BRIEF (3-5 sentences max). No headers, no bullet points, no explicit citations.
 
 Validate these as universal human needs. Share one insight about why these needs matter in connection. Optionally suggest what a request might sound like. Keep it warm and conversational - building emotional intelligence, not lecturing.
 
 Draw wisdom from NVC, attachment theory, and relational psychology - but embed it naturally, don't name sources.`,
-                            verbose: isVerbose
-                        })
+                        verbose: isVerbose
+                    }, (chunk, acc) => {
+                        if (!textEl) {
+                            removeToolLoading();
+                            textEl = addToolMsg('tool', '', 'Needs');
+                        }
+                        textEl.innerHTML = formatText(acc);
                     });
-                    const data = await response.json();
-                    removeToolLoading();
-                    if (data.response) {
-                        addToolMsg('tool', data.response, 'Needs');
-                    }
                 } catch (e) {
                     removeToolLoading();
-                    addToolMsg('tool', 'Connection error.', 'Error');
+                    if (textEl) { textEl.innerHTML = formatText('Connection error.'); }
+                    else { addToolMsg('tool', 'Connection error.', 'Error'); }
                 }
                 input.value = '';
                 return;
@@ -1700,15 +1727,13 @@ Draw wisdom from NVC, attachment theory, and relational psychology - but embed i
 
                 addToolLoading();
                 const isVerbose = document.getElementById('toggle-verbose')?.checked || false;
+                let textEl = null;
                 try {
-                    const response = await fetch('/api/tool', {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify({
-                            tool: 'wisdom',
-                            partner_message: lastPartnerMessage,
-                            user_draft: userInput || '',
-                            user_input: `The user draws wisdom from: ${selectedTradition}.
+                    await streamFetch('/api/tool', {
+                        tool: 'wisdom',
+                        partner_message: lastPartnerMessage,
+                        user_draft: userInput || '',
+                        user_input: `The user draws wisdom from: ${selectedTradition}.
 
 Context:
 ${contextText}
@@ -1716,16 +1741,17 @@ ${contextText}
 ${userInput ? 'User adds: ' + userInput : ''}
 
 Based on the themes here (conflict, connection, fear, love, boundaries, etc.), provide ONE relevant quote from ${selectedTradition} that speaks to this situation.`,
-                            verbose: isVerbose
-                        })
+                        verbose: isVerbose
+                    }, (chunk, acc) => {
+                        if (!textEl) {
+                            removeToolLoading();
+                            textEl = addToolMsg('tool', '', `${selectedTradition} Wisdom`);
+                        }
+                        textEl.innerHTML = formatText(acc);
                     });
-                    const data = await response.json();
-                    removeToolLoading();
-                    if (data.response) {
-                        addToolMsg('tool', data.response, `${selectedTradition} Wisdom`);
-                    }
                 } catch (e) {
                     removeToolLoading();
+                    if (textEl) { textEl.innerHTML = formatText('Connection error.'); return; }
                     addToolMsg('tool', 'Connection error.', 'Error');
                 }
                 input.value = '';
@@ -1820,28 +1846,27 @@ Based on the themes here (conflict, connection, fear, love, boundaries, etc.), p
             // Partner starts
             if (context) {
                 addLoading();
+                let textEl = null;
                 try {
-                    const response = await fetch('/api/chat', {
-                        method: 'POST',
-                        headers: { 'Content-Type': 'application/json' },
-                        body: JSON.stringify({
-                            messages: [{ role: 'user', content: 'Start based on: ' + context }],
-                            system: PARTNER_PROMPTS[style][difficulty] + ' CONTEXT: ' + context,
-                            max_tokens: 200
-                        })
+                    const full = await streamFetch('/api/chat', {
+                        messages: [{ role: 'user', content: 'Start based on: ' + context }],
+                        system: PARTNER_PROMPTS[style][difficulty] + ' CONTEXT: ' + context,
+                        max_tokens: 200
+                    }, (chunk, acc) => {
+                        if (!textEl) {
+                            removeLoading();
+                            textEl = addMessage('partner', '');
+                        }
+                        textEl.innerHTML = formatText(acc);
                     });
-                    const data = await response.json();
-                    removeLoading();
-                    if (data.content?.[0]) {
-                        const msg = data.content[0].text;
-                        addMessage('partner', msg);
-                        lastPartnerMessage = msg;
-                        conversationHistory.push({ role: 'assistant', content: msg });
+                    if (full) {
+                        lastPartnerMessage = full;
+                        conversationHistory.push({ role: 'assistant', content: full });
                     }
                 } catch (e) {
                     removeLoading();
                     const msg = OPENING_MESSAGES[style][difficulty];
-                    addMessage('partner', msg);
+                    if (textEl) { textEl.innerHTML = formatText(msg); } else { addMessage('partner', msg); }
                     lastPartnerMessage = msg;
                     conversationHistory.push({ role: 'assistant', content: msg });
                 }
@@ -1878,28 +1903,26 @@ Based on the themes here (conflict, connection, fear, love, boundaries, etc.), p
             conversationHistory.push({ role: 'user', content: message });
 
             addLoading();
+            let textEl = null;
             try {
                 const style = document.getElementById('attachment-style').value;
                 const difficulty = document.getElementById('difficulty').value;
 
-                const response = await fetch('/api/chat', {
-                    method: 'POST',
-                    headers: { 'Content-Type': 'application/json' },
-                    body: JSON.stringify({
-                        messages: conversationHistory,
-                        system: PARTNER_PROMPTS[style][difficulty],
-                        max_tokens: 300
-                    })
+                const full = await streamFetch('/api/chat', {
+                    messages: conversationHistory,
+                    system: PARTNER_PROMPTS[style][difficulty],
+                    max_tokens: 300
+                }, (chunk, acc) => {
+                    if (!textEl) {
+                        removeLoading();
+                        textEl = addMessage('partner', '');
+                    }
+                    textEl.innerHTML = formatText(acc);
                 });
 
-                const data = await response.json();
-                removeLoading();
-
-                if (data.content?.[0]) {
-                    const partnerMsg = data.content[0].text;
-                    addMessage('partner', partnerMsg);
-                    lastPartnerMessage = partnerMsg;
-                    conversationHistory.push({ role: 'assistant', content: partnerMsg });
+                if (full) {
+                    lastPartnerMessage = full;
+                    conversationHistory.push({ role: 'assistant', content: full });
                 }
             } catch (error) {
                 removeLoading();
@@ -1961,6 +1984,7 @@ Based on the themes here (conflict, connection, fear, love, boundaries, etc.), p
 
             conv.appendChild(div);
             conv.scrollTop = conv.scrollHeight;
+            return text;
         }
 
         function addToolMsg(type, content, label = null) {
@@ -1979,6 +2003,7 @@ Based on the themes here (conflict, connection, fear, love, boundaries, etc.), p
 
             cont.appendChild(div);
             cont.scrollTop = cont.scrollHeight;
+            return text;
         }
 
         function addLoading() {