HomePilot Deploy Bot commited on
Commit Β·
0ef8bc0
1
Parent(s): 4b3b11c
chore(hf): sync HomePilot to HF Space
Browse filesThis view is limited to 50 files because it contains too many changes. Β See raw diff
- .gitattributes +5 -5
- backend/app/main.py +38 -0
- backend/app/persona_call/__init__.py +33 -0
- backend/app/persona_call/backchannel.py +122 -0
- backend/app/persona_call/closing.py +102 -0
- backend/app/persona_call/config.py +102 -0
- backend/app/persona_call/context.py +191 -0
- backend/app/persona_call/directive.py +365 -0
- backend/app/persona_call/facets.py +180 -0
- backend/app/persona_call/latency.py +107 -0
- backend/app/persona_call/openings.py +334 -0
- backend/app/persona_call/repeat.py +128 -0
- backend/app/persona_call/router.py +224 -0
- backend/app/persona_call/state.py +99 -0
- backend/app/persona_call/store.py +241 -0
- backend/app/studio/render_jobs.py +14 -0
- backend/app/studio/render_mp4.py +206 -4
- backend/app/studio/routes.py +98 -0
- backend/app/voice_call/__init__.py +16 -0
- backend/app/voice_call/barge_in.py +123 -0
- backend/app/voice_call/config.py +92 -0
- backend/app/voice_call/models.py +112 -0
- backend/app/voice_call/policy.py +73 -0
- backend/app/voice_call/router.py +223 -0
- backend/app/voice_call/service.py +145 -0
- backend/app/voice_call/store.py +293 -0
- backend/app/voice_call/turn.py +125 -0
- backend/app/voice_call/turn_stream.py +259 -0
- backend/app/voice_call/ws.py +605 -0
- frontend/.cache-bust +1 -1
- frontend/src/ui/App.tsx +291 -8
- frontend/src/ui/CallOverlay.tsx +1389 -0
- frontend/src/ui/CreatorStudioEditor.tsx +6 -0
- frontend/src/ui/SettingsPanel.tsx +48 -20
- frontend/src/ui/call/bargeIn.test.ts +81 -0
- frontend/src/ui/call/bargeIn.ts +132 -0
- frontend/src/ui/call/callApi.ts +190 -0
- frontend/src/ui/call/callSocket.ts +579 -0
- frontend/src/ui/call/log.ts +140 -0
- frontend/src/ui/call/streamTts.test.ts +65 -0
- frontend/src/ui/call/streamTts.ts +173 -0
- frontend/src/ui/call/useCallSession.ts +381 -0
- frontend/src/ui/components/CreatorExportWizard.tsx +423 -65
- frontend/src/ui/components/TtsEngineSection.tsx +279 -0
- frontend/src/ui/lib/backendUrl.test.ts +35 -0
- frontend/src/ui/lib/backendUrl.ts +19 -2
- frontend/src/ui/phone/PostCallCard.test.tsx +156 -0
- frontend/src/ui/phone/PostCallCard.tsx +384 -0
- frontend/src/ui/phone/README.md +103 -0
- frontend/src/ui/phone/icons.tsx +144 -0
.gitattributes
CHANGED
|
@@ -1,9 +1,9 @@
|
|
| 1 |
-
*.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 2 |
-
*.woff filter=lfs diff=lfs merge=lfs -text
|
| 3 |
-
*.ico filter=lfs diff=lfs merge=lfs -text
|
| 4 |
*.hpersona filter=lfs diff=lfs merge=lfs -text
|
| 5 |
-
*.png filter=lfs diff=lfs merge=lfs -text
|
| 6 |
*.jpg filter=lfs diff=lfs merge=lfs -text
|
| 7 |
-
*.webp filter=lfs diff=lfs merge=lfs -text
|
| 8 |
*.svg filter=lfs diff=lfs merge=lfs -text
|
| 9 |
*.woff2 filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
*.hpersona filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 2 |
*.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 3 |
*.svg filter=lfs diff=lfs merge=lfs -text
|
| 4 |
*.woff2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ico filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.webp filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.woff filter=lfs diff=lfs merge=lfs -text
|
backend/app/main.py
CHANGED
|
@@ -159,6 +159,44 @@ app.include_router(teams_router)
|
|
| 159 |
from .teams.bridge_routes import router as teams_bridge_router
|
| 160 |
app.include_router(teams_bridge_router)
|
| 161 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 162 |
# Include Marketplace routes (/v1/marketplace/*)
|
| 163 |
app.include_router(marketplace_router)
|
| 164 |
|
|
|
|
| 159 |
from .teams.bridge_routes import router as teams_bridge_router
|
| 160 |
app.include_router(teams_bridge_router)
|
| 161 |
|
| 162 |
+
# --- Voice call (/v1/voice-call/*) β ADDITIVE, FEATURE-FLAGGED ---------------
|
| 163 |
+
# Fully off by default. Enable with ``VOICE_CALL_ENABLED=true``. If anything
|
| 164 |
+
# in the voice_call module raises on import (bad migration, missing dep, β¦)
|
| 165 |
+
# we swallow it so the rest of the app stays up β this feature is optional
|
| 166 |
+
# and must never take production down.
|
| 167 |
+
try:
|
| 168 |
+
from .voice_call.config import load as _voice_call_load_config
|
| 169 |
+
_voice_call_cfg = _voice_call_load_config()
|
| 170 |
+
if _voice_call_cfg.enabled:
|
| 171 |
+
from .voice_call.router import build_router as _build_vc_router
|
| 172 |
+
from .voice_call.router import build_ws_router as _build_vc_ws_router
|
| 173 |
+
app.include_router(_build_vc_router(_voice_call_cfg))
|
| 174 |
+
app.include_router(_build_vc_ws_router(_voice_call_cfg))
|
| 175 |
+
print("[voice_call] enabled β routes mounted under /v1/voice-call")
|
| 176 |
+
except Exception as _vc_err: # noqa: BLE001
|
| 177 |
+
# Deliberate broad catch: this entire feature is optional. Log and move on.
|
| 178 |
+
print(f"[voice_call] DISABLED due to import error: {_vc_err}")
|
| 179 |
+
|
| 180 |
+
# --- Persona call (/v1/persona-call/*) β ADDITIVE, FEATURE-FLAGGED -----------
|
| 181 |
+
# Phone-call conversational dynamics + per-turn system suffix composer.
|
| 182 |
+
# Off by default. Enable with ``PERSONA_CALL_ENABLED=true``. The module
|
| 183 |
+
# operates behind voice_call/ws.py β the HTTP router is minor (facets
|
| 184 |
+
# lookup, state inspection, shadow-directive audit). Independent flag so
|
| 185 |
+
# it can ship in shadow-compose mode (``PERSONA_CALL_APPLY=false``)
|
| 186 |
+
# before any prompt surface actually changes.
|
| 187 |
+
try:
|
| 188 |
+
from .persona_call.config import load as _persona_call_load_config
|
| 189 |
+
_persona_call_cfg = _persona_call_load_config()
|
| 190 |
+
if _persona_call_cfg.enabled:
|
| 191 |
+
from .persona_call.router import build_router as _build_pc_router
|
| 192 |
+
app.include_router(_build_pc_router(_persona_call_cfg))
|
| 193 |
+
print(
|
| 194 |
+
f"[persona_call] enabled (apply={_persona_call_cfg.apply}) β "
|
| 195 |
+
"routes mounted under /v1/persona-call"
|
| 196 |
+
)
|
| 197 |
+
except Exception as _pc_err: # noqa: BLE001
|
| 198 |
+
print(f"[persona_call] DISABLED due to import error: {_pc_err}")
|
| 199 |
+
|
| 200 |
# Include Marketplace routes (/v1/marketplace/*)
|
| 201 |
app.include_router(marketplace_router)
|
| 202 |
|
backend/app/persona_call/__init__.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
persona_call β phone-call conversational dynamics on top of voice_call.
|
| 3 |
+
|
| 4 |
+
Adds nothing to a persona's BASE system prompt. Instead, when a persona
|
| 5 |
+
is on a live phone call, this module composes a small per-turn system
|
| 6 |
+
*suffix* that:
|
| 7 |
+
|
| 8 |
+
* tells the persona it is currently on a phone call (not chat),
|
| 9 |
+
* applies one research-grounded rule from the design doc (e.g.
|
| 10 |
+
"skip 'how are you' after 9 pm local"),
|
| 11 |
+
* enforces the two-phase closing handshake at the state machine level
|
| 12 |
+
(the only place the model is overridden rather than advised),
|
| 13 |
+
* emits server-originated *backchannels* and *fillers* on a separate
|
| 14 |
+
low-volume stream so the user hears continuers + thinking sounds
|
| 15 |
+
without waiting for a full LLM turn.
|
| 16 |
+
|
| 17 |
+
Fully additive + feature-flagged. With ``PERSONA_CALL_ENABLED=false``
|
| 18 |
+
(the default) this module is never imported and no row is ever written
|
| 19 |
+
to the ``persona_call_state`` table.
|
| 20 |
+
|
| 21 |
+
Published research baked in:
|
| 22 |
+
Schegloff 1968, Schegloff & Sacks 1973, Sacks/Schegloff/Jefferson 1974,
|
| 23 |
+
Jefferson 1984, Stivers et al. 2009, Clark & French 1981,
|
| 24 |
+
Local & Walker 2012, OpenAI Realtime Prompting Guide, Sesame 2025.
|
| 25 |
+
|
| 26 |
+
Call-centre software-engineering practice applied:
|
| 27 |
+
single-source-of-truth state machine; every behavior has a named
|
| 28 |
+
module and a test anchor; deterministic telemetry counters for
|
| 29 |
+
naturalness SLOs; zero coupling to persona storage.
|
| 30 |
+
"""
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
__all__ = ["config", "directive", "router"]
|
backend/app/persona_call/backchannel.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Server-originated backchannel emission β the separate low-volume
|
| 3 |
+
stream (per the user's chosen default).
|
| 4 |
+
|
| 5 |
+
A backchannel is NOT a turn. It's a "mm-hm" / "yeah" / "got it" the
|
| 6 |
+
server emits *while the user is still speaking* so the caller hears
|
| 7 |
+
recipiency instead of silence. Research (Jefferson 1984) shows humans
|
| 8 |
+
emit one every 2β3 clauses on the phone β they compensate for the
|
| 9 |
+
absence of visual nods.
|
| 10 |
+
|
| 11 |
+
Design rules baked in:
|
| 12 |
+
|
| 13 |
+
* Token pulled from the persona's own pool (VoiceFacets.ack_tokens)
|
| 14 |
+
so Darkangel666 sounds like Darkangel666 even when backchanneling.
|
| 15 |
+
* Cadence: at most one per ``backchannel_min_gap_ms``, and only after
|
| 16 |
+
``backchannel_clause_trigger`` clause boundaries in the CURRENT
|
| 17 |
+
user utterance.
|
| 18 |
+
* Anti-repetition: never the same token two backchannels in a row.
|
| 19 |
+
* Played at ``backchannel_volume_db`` below the main TTS stream so
|
| 20 |
+
it overlaps the user's voice without fighting it (the client
|
| 21 |
+
applies the gain β this module just names the dB).
|
| 22 |
+
* Skipped entirely when ``facets.backchannel_rate`` is 0.
|
| 23 |
+
|
| 24 |
+
The emission itself is a WS event; this module picks the token and
|
| 25 |
+
decides IF/WHEN. The actual ``ws.send`` call lives in ws.py.
|
| 26 |
+
"""
|
| 27 |
+
from __future__ import annotations
|
| 28 |
+
|
| 29 |
+
import random
|
| 30 |
+
import re
|
| 31 |
+
import time
|
| 32 |
+
from typing import Any, Dict, List, Optional, Tuple
|
| 33 |
+
|
| 34 |
+
from .config import PersonaCallConfig
|
| 35 |
+
from .facets import VoiceFacets
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
# Rough clause boundaries β commas, semicolons, sentence enders,
|
| 39 |
+
# conjunctions that start a clause ("and thenβ¦", "butβ¦", "soβ¦").
|
| 40 |
+
# Runs on the partial STT transcript, so it fires as the user speaks.
|
| 41 |
+
_CLAUSE_BOUNDARY = re.compile(
|
| 42 |
+
r"("
|
| 43 |
+
r"[,;]|[.!?]\s+|" # punctuation
|
| 44 |
+
r"\b(and|but|so|because|then|however|actually)\b\s" # discourse markers
|
| 45 |
+
r")",
|
| 46 |
+
re.I,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def count_clause_boundaries(partial_text: str) -> int:
|
| 51 |
+
"""How many clause boundaries have been uttered in the current
|
| 52 |
+
user turn so far."""
|
| 53 |
+
if not partial_text:
|
| 54 |
+
return 0
|
| 55 |
+
return len(_CLAUSE_BOUNDARY.findall(partial_text))
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def _now_ms() -> int:
|
| 59 |
+
return int(time.time() * 1000)
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def choose_token(
|
| 63 |
+
facets: VoiceFacets,
|
| 64 |
+
*,
|
| 65 |
+
recent_acks: List[str],
|
| 66 |
+
seed_utterance: str = "",
|
| 67 |
+
) -> Optional[str]:
|
| 68 |
+
"""Pick a backchannel token that is (a) in the persona's pool and
|
| 69 |
+
(b) not the most recent one. Returns None iff the pool is empty.
|
| 70 |
+
|
| 71 |
+
Mild stance-sensitivity: if the partial utterance ended with a
|
| 72 |
+
rising question mark, we bias toward stance tokens ('oh really?',
|
| 73 |
+
'yeah?'); falling-period endings bias toward acknowledgments.
|
| 74 |
+
"""
|
| 75 |
+
pool = [t for t in facets.ack_tokens if t]
|
| 76 |
+
if not pool:
|
| 77 |
+
return None
|
| 78 |
+
last = (recent_acks[-1].lower() if recent_acks else "")
|
| 79 |
+
candidates = [t for t in pool if t.lower() != last] or pool
|
| 80 |
+
# Stance nudge β prefer single-syllable quick tokens on
|
| 81 |
+
# exclamations / questions, longer ones on plain pauses.
|
| 82 |
+
is_query = bool(seed_utterance.rstrip().endswith(("?", "!")))
|
| 83 |
+
if is_query:
|
| 84 |
+
candidates = sorted(candidates, key=lambda t: len(t))
|
| 85 |
+
return random.choice(candidates)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def should_emit(
|
| 89 |
+
*,
|
| 90 |
+
partial_text: str,
|
| 91 |
+
last_backchannel_ms: int,
|
| 92 |
+
facets: VoiceFacets,
|
| 93 |
+
cfg: PersonaCallConfig,
|
| 94 |
+
) -> bool:
|
| 95 |
+
"""True iff we should send a backchannel right now. Pure decision;
|
| 96 |
+
no side effects."""
|
| 97 |
+
if facets.backchannel_rate <= 0:
|
| 98 |
+
return False
|
| 99 |
+
# Scale the clause threshold by the persona's own rate. Low-rate
|
| 100 |
+
# personas (e.g. reserved / clinical) need more clauses before a
|
| 101 |
+
# backchannel; chatty personas need fewer.
|
| 102 |
+
trigger = max(
|
| 103 |
+
1, int(round(cfg.backchannel_clause_trigger / max(0.25, facets.backchannel_rate)))
|
| 104 |
+
)
|
| 105 |
+
if count_clause_boundaries(partial_text) < trigger:
|
| 106 |
+
return False
|
| 107 |
+
min_gap = int(cfg.backchannel_min_gap_ms / max(0.25, facets.backchannel_rate))
|
| 108 |
+
if _now_ms() - int(last_backchannel_ms or 0) < min_gap:
|
| 109 |
+
return False
|
| 110 |
+
return True
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def build_event(token: str, volume_db: float) -> Dict[str, Any]:
|
| 114 |
+
"""Shape the WS event payload ws.py will serialize."""
|
| 115 |
+
return {
|
| 116 |
+
"type": "assistant.backchannel",
|
| 117 |
+
"payload": {
|
| 118 |
+
"token": token,
|
| 119 |
+
"volume_db": float(volume_db),
|
| 120 |
+
"ts": _now_ms(),
|
| 121 |
+
},
|
| 122 |
+
}
|
backend/app/persona_call/closing.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Two-phase closing handshake β the one place we *enforce* instead of
|
| 3 |
+
advise.
|
| 4 |
+
|
| 5 |
+
Why enforced: LLMs reliably collapse Schegloff's 3β6-turn closing
|
| 6 |
+
into "Goodbye!" the moment they hear an "ok". Humans don't. The
|
| 7 |
+
handshake is a short, mandatory state transition that keeps the
|
| 8 |
+
persona from hanging up before the caller does.
|
| 9 |
+
|
| 10 |
+
Flow (per Schegloff & Sacks 1973, *Opening up Closings*):
|
| 11 |
+
|
| 12 |
+
user says "ok" / "alright" / "I should go" / "thanks, that's all"
|
| 13 |
+
β
|
| 14 |
+
βΌ
|
| 15 |
+
state.phase := 'pre_closing'
|
| 16 |
+
pre_closing_trigger := <the user's token>
|
| 17 |
+
β
|
| 18 |
+
βΌ
|
| 19 |
+
persona reply directive FORCES a pre-closing token reply:
|
| 20 |
+
- one short softener from facets.pre_closing_tokens
|
| 21 |
+
- no "bye", no "goodbye", no "take care" yet
|
| 22 |
+
β
|
| 23 |
+
βΌ (next user turn)
|
| 24 |
+
if user says "bye" / "talk later" / "see ya":
|
| 25 |
+
persona reply directive FORCES a mirrored two-part close:
|
| 26 |
+
<softener>, <terminal>
|
| 27 |
+
e.g. "take care, bye"
|
| 28 |
+
else:
|
| 29 |
+
state stays in 'pre_closing'; persona speaks a normal turn.
|
| 30 |
+
"""
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import re
|
| 34 |
+
from typing import Optional
|
| 35 |
+
|
| 36 |
+
from .facets import VoiceFacets
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# User utterances that start a closing (pre-closing trigger).
|
| 40 |
+
# Kept tight to avoid false positives mid-conversation.
|
| 41 |
+
_PRE_CLOSE_PATTERNS = [
|
| 42 |
+
re.compile(p, re.I) for p in [
|
| 43 |
+
r"^\s*(ok|okay|alright|ok\s+then)\s*[.!?]?\s*$",
|
| 44 |
+
r"\b(i should (go|run|get back)|gotta (go|run)|need to go)\b",
|
| 45 |
+
r"\b(thanks[, ]+(that'?s|that is)\s+(all|it))\b",
|
| 46 |
+
r"\b(talk (to you )?(later|soon)|call (you )?(later|back))\b",
|
| 47 |
+
]
|
| 48 |
+
]
|
| 49 |
+
|
| 50 |
+
# Pure terminal β "bye"-family tokens that complete the exchange.
|
| 51 |
+
_TERMINAL_PATTERNS = [
|
| 52 |
+
re.compile(p, re.I) for p in [
|
| 53 |
+
r"^\s*(bye|good[\- ]?bye|later|see ya|see you|ciao)\s*[.!?]?\s*$",
|
| 54 |
+
r"\b(bye bye|byebye)\b",
|
| 55 |
+
]
|
| 56 |
+
]
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def is_pre_closing_trigger(user_text: str) -> bool:
|
| 60 |
+
"""True iff the user just began a closing sequence."""
|
| 61 |
+
if not user_text:
|
| 62 |
+
return False
|
| 63 |
+
return any(p.search(user_text) for p in _PRE_CLOSE_PATTERNS)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def is_terminal(user_text: str) -> bool:
|
| 67 |
+
"""True iff the user said goodbye β the second half of the
|
| 68 |
+
two-phase close. Matched INDEPENDENTLY of the pre-close trigger
|
| 69 |
+
so the state machine can skip straight to terminal if the user
|
| 70 |
+
blurts out "bye!" without a prior "ok"."""
|
| 71 |
+
if not user_text:
|
| 72 |
+
return False
|
| 73 |
+
return any(p.search(user_text) for p in _TERMINAL_PATTERNS)
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# ββ directive fragments for each phase βββββββββββββββββββββββββββββββ
|
| 77 |
+
|
| 78 |
+
def pre_closing_directive(facets: VoiceFacets, trigger: Optional[str]) -> str:
|
| 79 |
+
"""Directive the composer injects when we're in 'pre_closing'
|
| 80 |
+
but the user hasn't said 'bye' yet."""
|
| 81 |
+
pool = ", ".join(facets.pre_closing_tokens[:4])
|
| 82 |
+
trig = f'"{trigger}"' if trigger else "a closing cue"
|
| 83 |
+
return (
|
| 84 |
+
f"The caller offered a pre-closing ({trig}). "
|
| 85 |
+
"Reciprocate with ONE matching soft pre-closing token β "
|
| 86 |
+
f"something from: {pool}. Do NOT say 'goodbye' yet. "
|
| 87 |
+
"Keep the turn to β€ 7 words and leave the actual goodbye to "
|
| 88 |
+
"the caller. Research: Schegloff & Sacks 1973 β the terminal "
|
| 89 |
+
"exchange is the caller's to initiate."
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def terminal_directive(facets: VoiceFacets) -> str:
|
| 94 |
+
"""Directive for the actual bye-for-bye turn."""
|
| 95 |
+
softeners = ", ".join(facets.pre_closing_tokens[:3])
|
| 96 |
+
closings = ", ".join(facets.closing_tokens[:3])
|
| 97 |
+
return (
|
| 98 |
+
"The caller just said goodbye. Mirror them with a TWO-PART "
|
| 99 |
+
f"close: one softener (e.g. {softeners}) followed by one "
|
| 100 |
+
f"terminal (e.g. {closings}). Never a single bare 'goodbye'. "
|
| 101 |
+
"Keep it to β€ 6 words. After this turn the call ends."
|
| 102 |
+
)
|
backend/app/persona_call/config.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Feature flags + runtime knobs for persona_call.
|
| 3 |
+
|
| 4 |
+
Every knob is an env var with a safe default. The module ships off.
|
| 5 |
+
|
| 6 |
+
Two flags, not one, so "shadow compose" (Β§ rollout P1 in the design)
|
| 7 |
+
can run without changing any outgoing prompt:
|
| 8 |
+
|
| 9 |
+
PERSONA_CALL_ENABLED β master. When false, this module is never
|
| 10 |
+
loaded into the app.
|
| 11 |
+
PERSONA_CALL_APPLY β when ENABLED=true but APPLY=false, the
|
| 12 |
+
composer runs and its output is logged to
|
| 13 |
+
``persona_call_directives`` but NOT injected
|
| 14 |
+
as a system message. Lets us collect
|
| 15 |
+
production directives for a week before
|
| 16 |
+
letting them shape turns.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import os
|
| 21 |
+
from dataclasses import dataclass
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def _bool_env(name: str, default: bool) -> bool:
|
| 25 |
+
raw = os.getenv(name)
|
| 26 |
+
if raw is None:
|
| 27 |
+
return default
|
| 28 |
+
return raw.strip().lower() in ("1", "true", "yes", "on")
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _int_env(name: str, default: int) -> int:
|
| 32 |
+
raw = os.getenv(name)
|
| 33 |
+
if raw is None:
|
| 34 |
+
return default
|
| 35 |
+
try:
|
| 36 |
+
return int(raw)
|
| 37 |
+
except (TypeError, ValueError):
|
| 38 |
+
return default
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _float_env(name: str, default: float) -> float:
|
| 42 |
+
raw = os.getenv(name)
|
| 43 |
+
if raw is None:
|
| 44 |
+
return default
|
| 45 |
+
try:
|
| 46 |
+
return float(raw)
|
| 47 |
+
except (TypeError, ValueError):
|
| 48 |
+
return default
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
@dataclass(frozen=True)
|
| 52 |
+
class PersonaCallConfig:
|
| 53 |
+
"""Resolved at every :func:`load` call β cheap, keep fresh for tests."""
|
| 54 |
+
|
| 55 |
+
# Master flag. False = this module is not imported into main.py.
|
| 56 |
+
enabled: bool
|
| 57 |
+
# When True, the composed directive is actually sent to the model.
|
| 58 |
+
# When False, it's logged only (shadow mode).
|
| 59 |
+
apply: bool
|
| 60 |
+
|
| 61 |
+
# Backchannel stream thresholds.
|
| 62 |
+
backchannel_clause_trigger: int # emit after N clause boundaries
|
| 63 |
+
backchannel_min_gap_ms: int # don't emit more often than this
|
| 64 |
+
backchannel_volume_db: float # played at -6 dB by default
|
| 65 |
+
|
| 66 |
+
# Filler emission during tool/LLM wait (Stivers 700 ms trouble line).
|
| 67 |
+
filler_emit_after_ms: int # default 600
|
| 68 |
+
|
| 69 |
+
# Phase machine thresholds.
|
| 70 |
+
how_are_you_late_hour: int # skip "how are you" at or after
|
| 71 |
+
late_night_brevity_hour: int # enter brevity bias at or after
|
| 72 |
+
late_night_brevity_hour_end: int # exit brevity bias at or before
|
| 73 |
+
|
| 74 |
+
# Anti-repetition ledger sizes.
|
| 75 |
+
ack_ledger_window: int
|
| 76 |
+
opener_ledger_window: int
|
| 77 |
+
|
| 78 |
+
# Closing handshake β the one enforced override.
|
| 79 |
+
closing_trigger_tokens_extra: str # comma-separated, user-supplied
|
| 80 |
+
|
| 81 |
+
# Reason-for-call fallback turn.
|
| 82 |
+
reason_fallback_turn: int # at this turn, ask if nothing heard
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def load() -> PersonaCallConfig:
|
| 86 |
+
return PersonaCallConfig(
|
| 87 |
+
enabled=_bool_env("PERSONA_CALL_ENABLED", False),
|
| 88 |
+
apply=_bool_env("PERSONA_CALL_APPLY", True),
|
| 89 |
+
backchannel_clause_trigger=_int_env("PERSONA_CALL_BC_CLAUSES", 2),
|
| 90 |
+
backchannel_min_gap_ms=_int_env("PERSONA_CALL_BC_MIN_GAP_MS", 1800),
|
| 91 |
+
backchannel_volume_db=_float_env("PERSONA_CALL_BC_VOL_DB", -6.0),
|
| 92 |
+
filler_emit_after_ms=_int_env("PERSONA_CALL_FILLER_AFTER_MS", 600),
|
| 93 |
+
how_are_you_late_hour=_int_env("PERSONA_CALL_HAY_LATE_HOUR", 21),
|
| 94 |
+
late_night_brevity_hour=_int_env("PERSONA_CALL_LATE_HOUR", 22),
|
| 95 |
+
late_night_brevity_hour_end=_int_env("PERSONA_CALL_LATE_HOUR_END", 6),
|
| 96 |
+
ack_ledger_window=_int_env("PERSONA_CALL_ACK_WINDOW", 5),
|
| 97 |
+
opener_ledger_window=_int_env("PERSONA_CALL_OPENER_WINDOW", 3),
|
| 98 |
+
closing_trigger_tokens_extra=os.getenv(
|
| 99 |
+
"PERSONA_CALL_CLOSING_EXTRA_TOKENS", ""
|
| 100 |
+
),
|
| 101 |
+
reason_fallback_turn=_int_env("PERSONA_CALL_REASON_FALLBACK_TURN", 3),
|
| 102 |
+
)
|
backend/app/persona_call/context.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Environmental + caller-context inputs for the per-turn composer.
|
| 3 |
+
|
| 4 |
+
Three kinds of signal, all computed on demand (never stored):
|
| 5 |
+
|
| 6 |
+
(a) Local clock β hour, day-of-week, weekend bit. Resolved from the
|
| 7 |
+
caller's ``device_info.tz`` when present, else server tz, else UTC.
|
| 8 |
+
Late-night brevity and morning-energy rules key off this.
|
| 9 |
+
|
| 10 |
+
(b) Recency β weeks since this (user, persona) last spoke. Drives the
|
| 11 |
+
"hey, it's been a while" one-shot acknowledgment.
|
| 12 |
+
|
| 13 |
+
(c) Caller-state extraction from STT β a tiny regex/keyword classifier
|
| 14 |
+
that runs on every user utterance:
|
| 15 |
+
* mobility (driving / walking / on transit)
|
| 16 |
+
* time pressure (quick question, running late, stepping out)
|
| 17 |
+
* reason-for-call sentence (turn β€ 3)
|
| 18 |
+
|
| 19 |
+
All inference is local + deterministic. No LLM calls in this module.
|
| 20 |
+
"""
|
| 21 |
+
from __future__ import annotations
|
| 22 |
+
|
| 23 |
+
import datetime as _dt
|
| 24 |
+
import re
|
| 25 |
+
from dataclasses import dataclass, field
|
| 26 |
+
from typing import Any, Dict, Optional
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
# ββ local clock βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 30 |
+
|
| 31 |
+
@dataclass(frozen=True)
|
| 32 |
+
class CallEnv:
|
| 33 |
+
local_hour: int
|
| 34 |
+
day_of_week: int # 0 Mon β¦ 6 Sun
|
| 35 |
+
is_weekend: bool
|
| 36 |
+
weeks_since_last_call: int # -1 when never
|
| 37 |
+
had_unfinished_topic: bool
|
| 38 |
+
resumed_from_topic: Optional[str] = None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def _resolve_tz(tz_name: Optional[str]):
|
| 42 |
+
"""Best-effort tz resolution. Returns a tzinfo or UTC."""
|
| 43 |
+
if not tz_name:
|
| 44 |
+
return _dt.timezone.utc
|
| 45 |
+
try:
|
| 46 |
+
# Python 3.9+ stdlib
|
| 47 |
+
from zoneinfo import ZoneInfo
|
| 48 |
+
return ZoneInfo(tz_name)
|
| 49 |
+
except Exception:
|
| 50 |
+
return _dt.timezone.utc
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def compute_env(
|
| 54 |
+
*,
|
| 55 |
+
tz: Optional[str] = None,
|
| 56 |
+
weeks_since_last_call: int = -1,
|
| 57 |
+
resumed_from_topic: Optional[str] = None,
|
| 58 |
+
now: Optional[_dt.datetime] = None,
|
| 59 |
+
) -> CallEnv:
|
| 60 |
+
"""Build a CallEnv from primitive inputs. No DB access here β
|
| 61 |
+
ws.py does the session lookups and calls this with the results so
|
| 62 |
+
this function stays cheap and unit-testable."""
|
| 63 |
+
tzinfo = _resolve_tz(tz)
|
| 64 |
+
current = (now or _dt.datetime.now(tz=_dt.timezone.utc)).astimezone(tzinfo)
|
| 65 |
+
return CallEnv(
|
| 66 |
+
local_hour=current.hour,
|
| 67 |
+
day_of_week=current.weekday(),
|
| 68 |
+
is_weekend=current.weekday() >= 5,
|
| 69 |
+
weeks_since_last_call=weeks_since_last_call,
|
| 70 |
+
had_unfinished_topic=bool(resumed_from_topic),
|
| 71 |
+
resumed_from_topic=resumed_from_topic,
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# ββ caller-state extraction βββββββββββββββββββββββββββββββββββββββββββ
|
| 76 |
+
|
| 77 |
+
# Deliberately narrow regexes. False positives in this module produce
|
| 78 |
+
# a directive like "the caller is driving β keep it to one sentence",
|
| 79 |
+
# which is always a safe fallback; false negatives lose the rule, which
|
| 80 |
+
# is also safe. So tight > comprehensive.
|
| 81 |
+
|
| 82 |
+
_MOBILITY_DRIVING = re.compile(
|
| 83 |
+
r"\b(i[' ]?m|i am|currently|just)?\s*(driving|behind the wheel|on the road|in the car)\b",
|
| 84 |
+
re.I,
|
| 85 |
+
)
|
| 86 |
+
_MOBILITY_WALKING = re.compile(
|
| 87 |
+
r"\b(i[' ]?m|i am)?\s*(walking|out on a walk|going for a walk)\b",
|
| 88 |
+
re.I,
|
| 89 |
+
)
|
| 90 |
+
_MOBILITY_TRANSIT = re.compile(
|
| 91 |
+
r"\b(on (the )?(train|subway|tube|metro|bus)|at the (station|airport)|boarding)\b",
|
| 92 |
+
re.I,
|
| 93 |
+
)
|
| 94 |
+
_TIME_PRESSURE = re.compile(
|
| 95 |
+
r"\b("
|
| 96 |
+
r"quick question|only have a minute|short on time|running late|"
|
| 97 |
+
r"stepping out|have to run|in a rush|in a hurry|gotta be quick"
|
| 98 |
+
r")\b",
|
| 99 |
+
re.I,
|
| 100 |
+
)
|
| 101 |
+
|
| 102 |
+
# Very loose reason-for-call cues for turn β€ 3. We don't need to
|
| 103 |
+
# extract a perfect paraphrase β we just need a yes/no flag so the
|
| 104 |
+
# directive composer can decide whether the phase machine can move
|
| 105 |
+
# out of 'opening'.
|
| 106 |
+
#
|
| 107 |
+
# Intentionally does NOT match "quick question" β that is a
|
| 108 |
+
# time-pressure signal, not an articulated reason. The composer reads
|
| 109 |
+
# both signals and will still skip 'how are you' on time pressure
|
| 110 |
+
# alone; we just don't want to prematurely promote phase β topic
|
| 111 |
+
# before the skip rule fires.
|
| 112 |
+
_REASON_CUE = re.compile(
|
| 113 |
+
r"\b("
|
| 114 |
+
r"i('?m| am)?\s+calling (about|to|because|for)|"
|
| 115 |
+
r"i wanted to (talk|ask|check|discuss|tell)|"
|
| 116 |
+
r"the reason i('?m| am) calling|"
|
| 117 |
+
r"i need (to know|help with|your help|a hand)"
|
| 118 |
+
r")\b",
|
| 119 |
+
re.I,
|
| 120 |
+
)
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
@dataclass
|
| 124 |
+
class CallerSignal:
|
| 125 |
+
driving: bool = False
|
| 126 |
+
walking: bool = False
|
| 127 |
+
transit: bool = False
|
| 128 |
+
time_pressured: bool = False
|
| 129 |
+
reason_expressed: bool = False
|
| 130 |
+
reason_text: str = ""
|
| 131 |
+
|
| 132 |
+
def merge(self, other: "CallerSignal") -> "CallerSignal":
|
| 133 |
+
"""Accumulate across multiple user utterances in one call.
|
| 134 |
+
Once a flag is set, it stays set until the phase machine
|
| 135 |
+
resets it."""
|
| 136 |
+
return CallerSignal(
|
| 137 |
+
driving=self.driving or other.driving,
|
| 138 |
+
walking=self.walking or other.walking,
|
| 139 |
+
transit=self.transit or other.transit,
|
| 140 |
+
time_pressured=self.time_pressured or other.time_pressured,
|
| 141 |
+
reason_expressed=self.reason_expressed or other.reason_expressed,
|
| 142 |
+
reason_text=self.reason_text or other.reason_text,
|
| 143 |
+
)
|
| 144 |
+
|
| 145 |
+
def to_dict(self) -> Dict[str, Any]:
|
| 146 |
+
return {
|
| 147 |
+
"driving": self.driving,
|
| 148 |
+
"walking": self.walking,
|
| 149 |
+
"transit": self.transit,
|
| 150 |
+
"time_pressured": self.time_pressured,
|
| 151 |
+
"reason_expressed": self.reason_expressed,
|
| 152 |
+
"reason_text": self.reason_text,
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
@classmethod
|
| 156 |
+
def from_dict(cls, d: Optional[Dict[str, Any]]) -> "CallerSignal":
|
| 157 |
+
d = d or {}
|
| 158 |
+
return cls(
|
| 159 |
+
driving=bool(d.get("driving")),
|
| 160 |
+
walking=bool(d.get("walking")),
|
| 161 |
+
transit=bool(d.get("transit")),
|
| 162 |
+
time_pressured=bool(d.get("time_pressured")),
|
| 163 |
+
reason_expressed=bool(d.get("reason_expressed")),
|
| 164 |
+
reason_text=str(d.get("reason_text") or ""),
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def classify_utterance(text: str) -> CallerSignal:
|
| 169 |
+
"""Extract a CallerSignal from one user turn. O(len(text)) regex."""
|
| 170 |
+
if not text:
|
| 171 |
+
return CallerSignal()
|
| 172 |
+
sig = CallerSignal(
|
| 173 |
+
driving=bool(_MOBILITY_DRIVING.search(text)),
|
| 174 |
+
walking=bool(_MOBILITY_WALKING.search(text)),
|
| 175 |
+
transit=bool(_MOBILITY_TRANSIT.search(text)),
|
| 176 |
+
time_pressured=bool(_TIME_PRESSURE.search(text)),
|
| 177 |
+
)
|
| 178 |
+
m = _REASON_CUE.search(text)
|
| 179 |
+
if m:
|
| 180 |
+
sig.reason_expressed = True
|
| 181 |
+
# Take the remainder of the sentence as the reason paraphrase
|
| 182 |
+
# β a cheap heuristic good enough for the phase machine.
|
| 183 |
+
start = m.end()
|
| 184 |
+
remainder = text[start:].strip()
|
| 185 |
+
end = len(remainder)
|
| 186 |
+
for stop in (".", "?", "!", "\n"):
|
| 187 |
+
idx = remainder.find(stop)
|
| 188 |
+
if idx != -1 and idx < end:
|
| 189 |
+
end = idx
|
| 190 |
+
sig.reason_text = remainder[:end].strip()[:140]
|
| 191 |
+
return sig
|
backend/app/persona_call/directive.py
ADDED
|
@@ -0,0 +1,365 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-turn system-suffix composer.
|
| 3 |
+
|
| 4 |
+
**Core invariant of this whole module** (the product promise):
|
| 5 |
+
|
| 6 |
+
The persona's own system prompt is NEVER touched. `compose()`
|
| 7 |
+
only produces a short suffix that is appended as a SEPARATE
|
| 8 |
+
system message in addition to whatever the /v1/chat/completions
|
| 9 |
+
endpoint already injects for the persona. When the call ends,
|
| 10 |
+
the suffix is gone; the persona's chat behavior is identical to
|
| 11 |
+
what it was before the call started.
|
| 12 |
+
|
| 13 |
+
The suffix is assembled from five small paragraphs. Each paragraph
|
| 14 |
+
is independently testable and individually gated by the current
|
| 15 |
+
phase / time / caller-context, so on an uneventful turn the suffix
|
| 16 |
+
is only ~4 lines long.
|
| 17 |
+
|
| 18 |
+
The five stages, in assembly order:
|
| 19 |
+
|
| 20 |
+
1. _base_phone_frame persona-neutral phone register
|
| 21 |
+
2. _phase_frame opening / topic / pre_closing / closed
|
| 22 |
+
3. _context_frame late-night / driving / resumed-topic
|
| 23 |
+
4. _repetition_frame ledger-enforced variety
|
| 24 |
+
5. _latency_frame filler rule for thinking pauses
|
| 25 |
+
"""
|
| 26 |
+
from __future__ import annotations
|
| 27 |
+
|
| 28 |
+
from dataclasses import dataclass, field
|
| 29 |
+
from typing import Any, Dict, List, Optional
|
| 30 |
+
|
| 31 |
+
from . import closing, repeat, state as state_mod, store
|
| 32 |
+
from .config import PersonaCallConfig
|
| 33 |
+
from .context import CallEnv, CallerSignal
|
| 34 |
+
from .facets import VoiceFacets
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ββ public result type βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 38 |
+
|
| 39 |
+
@dataclass
|
| 40 |
+
class ComposedDirective:
|
| 41 |
+
"""Return value of :func:`compose`. Two independently useful fields:
|
| 42 |
+
|
| 43 |
+
* ``system_suffix`` β the text we append as an additional system
|
| 44 |
+
message. If :class:`PersonaCallConfig`.apply is False (shadow
|
| 45 |
+
mode), this is logged but not actually sent to the model.
|
| 46 |
+
* ``post_directives`` β the list of individual rules we applied,
|
| 47 |
+
one string per rule, in the order composed. Dumped into
|
| 48 |
+
``persona_call_directives.post_directives`` for observability.
|
| 49 |
+
"""
|
| 50 |
+
session_id: str
|
| 51 |
+
phase: str
|
| 52 |
+
turn_index: int
|
| 53 |
+
system_suffix: str
|
| 54 |
+
post_directives: List[str] = field(default_factory=list)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ββ paragraph builders ββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 58 |
+
|
| 59 |
+
def _base_phone_frame(facets: VoiceFacets) -> str:
|
| 60 |
+
"""Persona-neutral phone register. Same for every persona; the
|
| 61 |
+
persona's identity is NOT asserted here β that stays in the
|
| 62 |
+
persona's own system prompt which is already active."""
|
| 63 |
+
return (
|
| 64 |
+
"[PHONE CALL MODE β appended only for the duration of this call]\n"
|
| 65 |
+
"You are on a live phone call. Spoken, not typed. "
|
| 66 |
+
"No bullet points, no stage directions (never output '*smiles*' "
|
| 67 |
+
"or similar), no markdown headers. "
|
| 68 |
+
f"Keep replies in the {facets.typical_reply_sentences}-sentence range, "
|
| 69 |
+
f"β€ {facets.sentence_max_words} words per sentence. "
|
| 70 |
+
f"Tone: {facets.tone_line}."
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _phase_frame(
|
| 75 |
+
*,
|
| 76 |
+
phase: str,
|
| 77 |
+
turn_index: int,
|
| 78 |
+
skipped_how_are_you: bool,
|
| 79 |
+
asked_reason_fallback: bool,
|
| 80 |
+
facets: VoiceFacets,
|
| 81 |
+
env: CallEnv,
|
| 82 |
+
caller: CallerSignal,
|
| 83 |
+
pre_closing_trigger: Optional[str],
|
| 84 |
+
cfg: PersonaCallConfig,
|
| 85 |
+
) -> tuple[str, List[str]]:
|
| 86 |
+
"""Return (paragraph, rule_names_applied) for the current phase."""
|
| 87 |
+
rules: List[str] = []
|
| 88 |
+
|
| 89 |
+
if phase == state_mod.OPENING:
|
| 90 |
+
# Turn 0 β Schegloff summons-answer.
|
| 91 |
+
if turn_index == 0:
|
| 92 |
+
rules.append("opening.summons_answer")
|
| 93 |
+
return (
|
| 94 |
+
"This is turn 0. Reply with ONE short line of recognition "
|
| 95 |
+
"β acknowledge you're there, use the caller's name if you "
|
| 96 |
+
"know it, β€ 6 words. Do NOT launch into topic.",
|
| 97 |
+
rules,
|
| 98 |
+
)
|
| 99 |
+
# Turn β₯ 1 β decide on "how are you" exactly once.
|
| 100 |
+
if not skipped_how_are_you and turn_index <= 2:
|
| 101 |
+
# Skip "how are you" if: late hour, caller is pressed, or
|
| 102 |
+
# caller already expressed a reason.
|
| 103 |
+
should_skip = (
|
| 104 |
+
env.local_hour >= cfg.how_are_you_late_hour
|
| 105 |
+
or caller.time_pressured
|
| 106 |
+
or caller.reason_expressed
|
| 107 |
+
)
|
| 108 |
+
if should_skip:
|
| 109 |
+
rules.append("opening.skip_how_are_you")
|
| 110 |
+
return (
|
| 111 |
+
"Skip the 'how are you?' exchange this time β the "
|
| 112 |
+
"caller is either pressed, late, or already on-topic. "
|
| 113 |
+
"Move directly to hearing them out.",
|
| 114 |
+
rules,
|
| 115 |
+
)
|
| 116 |
+
rules.append("opening.how_are_you_once")
|
| 117 |
+
return (
|
| 118 |
+
"One brief round of 'how are you?' is appropriate now. "
|
| 119 |
+
"Ask once, let them answer, then pause for their reason "
|
| 120 |
+
"for calling. Do not stack this with another question.",
|
| 121 |
+
rules,
|
| 122 |
+
)
|
| 123 |
+
# Opening past turn 2 with no reason and no fallback asked.
|
| 124 |
+
if (
|
| 125 |
+
not caller.reason_expressed
|
| 126 |
+
and not asked_reason_fallback
|
| 127 |
+
and turn_index >= cfg.reason_fallback_turn
|
| 128 |
+
):
|
| 129 |
+
rules.append("opening.reason_fallback_ask")
|
| 130 |
+
return (
|
| 131 |
+
"The caller has not said why they're calling yet. "
|
| 132 |
+
"Ask gently, in one sentence: something like "
|
| 133 |
+
"'what's on your mind today?' or 'what's up?'",
|
| 134 |
+
rules,
|
| 135 |
+
)
|
| 136 |
+
rules.append("opening.default")
|
| 137 |
+
return (
|
| 138 |
+
"Still in the opening. Keep replies short and oriented to "
|
| 139 |
+
"the caller's lead.",
|
| 140 |
+
rules,
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
if phase == state_mod.TOPIC:
|
| 144 |
+
rules.append("topic.default")
|
| 145 |
+
return (
|
| 146 |
+
"Mid-call. Be responsive: if the caller asked a question, "
|
| 147 |
+
"answer it before asking one back. Ask at most ONE follow-up "
|
| 148 |
+
"per turn β phone calls tolerate fewer questions per turn "
|
| 149 |
+
"than chat does.",
|
| 150 |
+
rules,
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
if phase == state_mod.PRE_CLOSING:
|
| 154 |
+
rules.append("closing.pre_closing_handshake")
|
| 155 |
+
return closing.pre_closing_directive(facets, pre_closing_trigger), rules
|
| 156 |
+
|
| 157 |
+
if phase == state_mod.CLOSED:
|
| 158 |
+
rules.append("closing.terminal")
|
| 159 |
+
return closing.terminal_directive(facets), rules
|
| 160 |
+
|
| 161 |
+
return "", rules
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
def _context_frame(
|
| 165 |
+
env: CallEnv, caller: CallerSignal, cfg: PersonaCallConfig,
|
| 166 |
+
) -> tuple[str, List[str]]:
|
| 167 |
+
"""Situational rules. Paragraphs only emit when the predicate fires,
|
| 168 |
+
so uneventful turns get no context frame at all."""
|
| 169 |
+
lines: List[str] = []
|
| 170 |
+
rules: List[str] = []
|
| 171 |
+
|
| 172 |
+
# Late-night brevity (decision-fatigue bias).
|
| 173 |
+
late = (
|
| 174 |
+
env.local_hour >= cfg.late_night_brevity_hour
|
| 175 |
+
or env.local_hour <= cfg.late_night_brevity_hour_end
|
| 176 |
+
)
|
| 177 |
+
if late:
|
| 178 |
+
rules.append("context.late_night_brevity")
|
| 179 |
+
lines.append(
|
| 180 |
+
f"Local clock: {env.local_hour:02d}:00 for the caller. "
|
| 181 |
+
"It's late β keep replies to one sentence when you can, "
|
| 182 |
+
"offer to continue tomorrow for anything complex."
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
# Driving β safety/attention bias.
|
| 186 |
+
if caller.driving:
|
| 187 |
+
rules.append("context.driving")
|
| 188 |
+
lines.append(
|
| 189 |
+
"The caller is DRIVING. One-sentence answers, no multi-part "
|
| 190 |
+
"questions, never require a yes/no decision unless it's "
|
| 191 |
+
"genuinely urgent."
|
| 192 |
+
)
|
| 193 |
+
elif caller.walking or caller.transit:
|
| 194 |
+
rules.append("context.mobile")
|
| 195 |
+
lines.append(
|
| 196 |
+
"The caller is mobile. Keep it brief and well-paced β they "
|
| 197 |
+
"may lose signal or attention momentarily."
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
# Weeks-since-last-call resumption (fire at most once per call; the
|
| 201 |
+
# persona_call_state doesn't need a separate flag because this only
|
| 202 |
+
# fires when the topic hasn't been acknowledged yet β the composer
|
| 203 |
+
# sees the same state each turn until the persona says something).
|
| 204 |
+
if env.weeks_since_last_call >= 3:
|
| 205 |
+
rules.append("context.resumption")
|
| 206 |
+
lines.append(
|
| 207 |
+
f"It has been β{env.weeks_since_last_call} weeks since this "
|
| 208 |
+
"caller last spoke with you. One natural acknowledgment of "
|
| 209 |
+
"the gap is appropriate β do NOT make it the whole turn, "
|
| 210 |
+
"and say it only once this call."
|
| 211 |
+
)
|
| 212 |
+
if env.resumed_from_topic:
|
| 213 |
+
rules.append("context.resume_topic")
|
| 214 |
+
lines.append(
|
| 215 |
+
f"Unfinished topic from a prior call: {env.resumed_from_topic!r}. "
|
| 216 |
+
"Offer to pick it back up if natural, but don't force it."
|
| 217 |
+
)
|
| 218 |
+
|
| 219 |
+
return ("\n".join(lines) if lines else ""), rules
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def _repetition_frame(
|
| 223 |
+
state: Dict[str, Any], facets: VoiceFacets,
|
| 224 |
+
) -> tuple[str, List[str]]:
|
| 225 |
+
text = repeat.forbid_clause(state, facets)
|
| 226 |
+
return (text, ["repeat.ledger"] if text else [])
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def _latency_frame(facets: VoiceFacets) -> tuple[str, List[str]]:
|
| 230 |
+
thinking = ", ".join(facets.thinking_tokens[:3])
|
| 231 |
+
return (
|
| 232 |
+
"If you need a beat to think, start your reply with a short "
|
| 233 |
+
f"filler from: {thinking}. Never produce a long silence β "
|
| 234 |
+
"callers read silence on the phone as trouble.",
|
| 235 |
+
["latency.filler_rule"],
|
| 236 |
+
)
|
| 237 |
+
|
| 238 |
+
|
| 239 |
+
# ββ public compose() ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 240 |
+
|
| 241 |
+
def compose(
|
| 242 |
+
*,
|
| 243 |
+
session_id: str,
|
| 244 |
+
persona_id: Optional[str],
|
| 245 |
+
user_text: str,
|
| 246 |
+
env: CallEnv,
|
| 247 |
+
cfg: PersonaCallConfig,
|
| 248 |
+
facets: Optional[VoiceFacets] = None,
|
| 249 |
+
) -> ComposedDirective:
|
| 250 |
+
"""Build the per-turn directive.
|
| 251 |
+
|
| 252 |
+
1. Advances the phase machine on the user utterance.
|
| 253 |
+
2. Composes the five paragraphs.
|
| 254 |
+
3. Persists the directive in shadow mode OR applies it.
|
| 255 |
+
"""
|
| 256 |
+
from .facets import for_persona_id
|
| 257 |
+
_facets = facets if facets is not None else for_persona_id(persona_id)
|
| 258 |
+
|
| 259 |
+
# Step 1: phase machine.
|
| 260 |
+
new_state = state_mod.advance_on_user_utterance(
|
| 261 |
+
session_id=session_id,
|
| 262 |
+
user_text=user_text,
|
| 263 |
+
reason_fallback_turn=cfg.reason_fallback_turn,
|
| 264 |
+
)
|
| 265 |
+
phase = str(new_state.get("phase") or state_mod.OPENING)
|
| 266 |
+
turn_index = int(new_state.get("turn_index") or 0)
|
| 267 |
+
|
| 268 |
+
# The caller_context column is already updated by the phase machine,
|
| 269 |
+
# so re-derive the CallerSignal from it for the composer.
|
| 270 |
+
caller = CallerSignal.from_dict(
|
| 271 |
+
new_state.get("caller_context")
|
| 272 |
+
if isinstance(new_state.get("caller_context"), dict) else {}
|
| 273 |
+
)
|
| 274 |
+
|
| 275 |
+
# Step 2: five paragraphs.
|
| 276 |
+
rules_all: List[str] = []
|
| 277 |
+
parts: List[str] = [_base_phone_frame(_facets)]
|
| 278 |
+
rules_all.append("base.phone_frame")
|
| 279 |
+
|
| 280 |
+
phase_para, r = _phase_frame(
|
| 281 |
+
phase=phase,
|
| 282 |
+
turn_index=turn_index - 1, # phase machine incremented already
|
| 283 |
+
skipped_how_are_you=bool(new_state.get("skipped_how_are_you")),
|
| 284 |
+
asked_reason_fallback=bool(new_state.get("asked_reason_fallback")),
|
| 285 |
+
facets=_facets,
|
| 286 |
+
env=env,
|
| 287 |
+
caller=caller,
|
| 288 |
+
pre_closing_trigger=(
|
| 289 |
+
str(new_state.get("pre_closing_trigger") or "")
|
| 290 |
+
if new_state.get("pre_closing_trigger") else None
|
| 291 |
+
),
|
| 292 |
+
cfg=cfg,
|
| 293 |
+
)
|
| 294 |
+
if phase_para:
|
| 295 |
+
parts.append(phase_para)
|
| 296 |
+
rules_all.extend(r)
|
| 297 |
+
|
| 298 |
+
ctx_para, r = _context_frame(env, caller, cfg)
|
| 299 |
+
if ctx_para:
|
| 300 |
+
parts.append(ctx_para)
|
| 301 |
+
rules_all.extend(r)
|
| 302 |
+
|
| 303 |
+
rep_para, r = _repetition_frame(new_state, _facets)
|
| 304 |
+
if rep_para:
|
| 305 |
+
parts.append(rep_para)
|
| 306 |
+
rules_all.extend(r)
|
| 307 |
+
|
| 308 |
+
lat_para, r = _latency_frame(_facets)
|
| 309 |
+
parts.append(lat_para)
|
| 310 |
+
rules_all.extend(r)
|
| 311 |
+
|
| 312 |
+
system_suffix = "\n\n".join(parts).strip()
|
| 313 |
+
|
| 314 |
+
# Step 3: persist + (maybe) apply.
|
| 315 |
+
store.record_directive(
|
| 316 |
+
session_id=session_id,
|
| 317 |
+
turn_index=turn_index,
|
| 318 |
+
phase=phase,
|
| 319 |
+
applied=cfg.apply,
|
| 320 |
+
system_suffix=system_suffix,
|
| 321 |
+
post_directives=rules_all,
|
| 322 |
+
)
|
| 323 |
+
|
| 324 |
+
# If we asked 'how are you' or the reason-fallback, set the flags
|
| 325 |
+
# so we don't do it again.
|
| 326 |
+
if "opening.how_are_you_once" in rules_all:
|
| 327 |
+
state_mod.mark_how_are_you_decision(session_id, skipped=False)
|
| 328 |
+
if "opening.skip_how_are_you" in rules_all:
|
| 329 |
+
state_mod.mark_how_are_you_decision(session_id, skipped=True)
|
| 330 |
+
if "opening.reason_fallback_ask" in rules_all:
|
| 331 |
+
state_mod.mark_reason_fallback_asked(session_id)
|
| 332 |
+
|
| 333 |
+
return ComposedDirective(
|
| 334 |
+
session_id=session_id,
|
| 335 |
+
phase=phase,
|
| 336 |
+
turn_index=turn_index,
|
| 337 |
+
system_suffix=system_suffix if cfg.apply else "",
|
| 338 |
+
post_directives=rules_all,
|
| 339 |
+
)
|
| 340 |
+
|
| 341 |
+
|
| 342 |
+
def record_persona_reply(
|
| 343 |
+
*,
|
| 344 |
+
session_id: str,
|
| 345 |
+
persona_id: Optional[str],
|
| 346 |
+
reply: str,
|
| 347 |
+
cfg: PersonaCallConfig,
|
| 348 |
+
) -> None:
|
| 349 |
+
"""Called after the LLM reply comes back. Rolls the anti-repetition
|
| 350 |
+
ledger forward so the next turn can forbid the just-used token."""
|
| 351 |
+
from .facets import for_persona_id
|
| 352 |
+
_facets = for_persona_id(persona_id)
|
| 353 |
+
state = store.ensure_state(session_id)
|
| 354 |
+
new_acks, new_openers = repeat.record_reply(
|
| 355 |
+
state,
|
| 356 |
+
reply,
|
| 357 |
+
_facets,
|
| 358 |
+
cfg_ack_window=cfg.ack_ledger_window,
|
| 359 |
+
cfg_opener_window=cfg.opener_ledger_window,
|
| 360 |
+
)
|
| 361 |
+
store.update_state(
|
| 362 |
+
session_id,
|
| 363 |
+
recent_acks=new_acks,
|
| 364 |
+
recent_openers=new_openers,
|
| 365 |
+
)
|
backend/app/persona_call/facets.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-persona ``voice_facets`` β non-destructive voice personalization.
|
| 3 |
+
|
| 4 |
+
Facets live in an OPTIONAL field on the persona manifest
|
| 5 |
+
(``persona.voice_facets``). Any existing persona with no facets gets
|
| 6 |
+
the neutral-English default β its chat behavior is completely
|
| 7 |
+
unchanged either way.
|
| 8 |
+
|
| 9 |
+
The facets object drives four things at runtime:
|
| 10 |
+
|
| 11 |
+
* backchannel token pools (``mm``, ``yeah``, ``right`` etc.)
|
| 12 |
+
* pre-closing + closing token pools
|
| 13 |
+
* filler / thinking tokens for the latency hook
|
| 14 |
+
* scalar voice knobs (warmth, pace, formality) interpolated into
|
| 15 |
+
the per-turn system suffix as raw numbers β the model reads the
|
| 16 |
+
number, not an adjective. This avoids the same three adjectives
|
| 17 |
+
bleeding into every reply.
|
| 18 |
+
|
| 19 |
+
The *base system prompt* of the persona is never touched. This file
|
| 20 |
+
never rewrites a persona; it only reads the optional facets field.
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
from dataclasses import dataclass, field
|
| 25 |
+
from typing import Any, Dict, List, Optional
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
# ββ Default facets β applied when a persona ships no voice_facets βββ
|
| 29 |
+
#
|
| 30 |
+
# Values chosen from the CA literature + common-sense phone norms.
|
| 31 |
+
# Token pools kept short on purpose: fewer options β easier anti-
|
| 32 |
+
# repetition tracking β cleaner entropy metric.
|
| 33 |
+
|
| 34 |
+
_DEFAULT_ACKS = ["mm", "mm-hm", "yeah", "right", "got it", "okay"]
|
| 35 |
+
_DEFAULT_THINKING = ["hmm", "let me think", "one sec", "hold on"]
|
| 36 |
+
_DEFAULT_PRE_CLOSE = ["okay", "alright", "so", "well"]
|
| 37 |
+
_DEFAULT_CLOSING = ["take care", "talk soon", "bye for now", "later"]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
@dataclass(frozen=True)
|
| 41 |
+
class VoiceFacets:
|
| 42 |
+
"""Immutable snapshot for one persona.
|
| 43 |
+
|
| 44 |
+
All scalar knobs are clamped on load so downstream code can trust
|
| 45 |
+
the range. Token pools are deduplicated + lowercased to keep the
|
| 46 |
+
anti-repetition ledger case-insensitive.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
warmth: float # 0 clinical β¦ 1 affectionate
|
| 50 |
+
pace: float # 0.25 β¦ 2.0 speech-rate multiplier
|
| 51 |
+
formality: float # 0 casual β¦ 1 formal
|
| 52 |
+
humor: float # 0 none β¦ 1 playful
|
| 53 |
+
self_disclosure: float # 0 reserved β¦ 1 volunteers a lot
|
| 54 |
+
sentence_max_words: int
|
| 55 |
+
typical_reply_sentences: List[int]
|
| 56 |
+
language_register: str
|
| 57 |
+
ack_tokens: List[str]
|
| 58 |
+
thinking_tokens: List[str]
|
| 59 |
+
pre_closing_tokens: List[str]
|
| 60 |
+
closing_tokens: List[str]
|
| 61 |
+
backchannel_rate: float # 0..2, multiplier on default cadence
|
| 62 |
+
|
| 63 |
+
@property
|
| 64 |
+
def tone_line(self) -> str:
|
| 65 |
+
"""One-line summary of scalar knobs, ready to drop into the
|
| 66 |
+
per-turn system suffix. Numeric so the model interpolates
|
| 67 |
+
rather than falling back on the same adjective."""
|
| 68 |
+
return (
|
| 69 |
+
f"warmth {self.warmth:.1f} Β· pace {self.pace:.2f} Β· "
|
| 70 |
+
f"formality {self.formality:.1f} Β· humor {self.humor:.1f}"
|
| 71 |
+
)
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _clamp(x: float, lo: float, hi: float) -> float:
|
| 75 |
+
try:
|
| 76 |
+
return max(lo, min(hi, float(x)))
|
| 77 |
+
except (TypeError, ValueError):
|
| 78 |
+
return lo
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def _tokens(raw: Any, default: List[str]) -> List[str]:
|
| 82 |
+
"""Normalize a token-pool field: list[str], lowercase, de-dup,
|
| 83 |
+
keep stable order. Returns the default if input is empty / bad."""
|
| 84 |
+
if not isinstance(raw, list) or not raw:
|
| 85 |
+
return list(default)
|
| 86 |
+
seen, out = set(), []
|
| 87 |
+
for item in raw:
|
| 88 |
+
if not isinstance(item, str):
|
| 89 |
+
continue
|
| 90 |
+
t = item.strip().lower()
|
| 91 |
+
if t and t not in seen:
|
| 92 |
+
seen.add(t)
|
| 93 |
+
out.append(t)
|
| 94 |
+
return out if out else list(default)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def default_facets() -> VoiceFacets:
|
| 98 |
+
"""Neutral English phone-call default. What an unmodified persona
|
| 99 |
+
gets β indistinguishable from today's behavior except for the phone
|
| 100 |
+
context itself."""
|
| 101 |
+
return VoiceFacets(
|
| 102 |
+
warmth=0.5,
|
| 103 |
+
pace=1.0,
|
| 104 |
+
formality=0.4,
|
| 105 |
+
humor=0.3,
|
| 106 |
+
self_disclosure=0.3,
|
| 107 |
+
sentence_max_words=22,
|
| 108 |
+
typical_reply_sentences=[1, 2, 3],
|
| 109 |
+
language_register="en-US-casual",
|
| 110 |
+
ack_tokens=list(_DEFAULT_ACKS),
|
| 111 |
+
thinking_tokens=list(_DEFAULT_THINKING),
|
| 112 |
+
pre_closing_tokens=list(_DEFAULT_PRE_CLOSE),
|
| 113 |
+
closing_tokens=list(_DEFAULT_CLOSING),
|
| 114 |
+
backchannel_rate=1.0,
|
| 115 |
+
)
|
| 116 |
+
|
| 117 |
+
|
| 118 |
+
def from_manifest(manifest: Optional[Dict[str, Any]]) -> VoiceFacets:
|
| 119 |
+
"""Parse a persona manifest dict into a VoiceFacets.
|
| 120 |
+
|
| 121 |
+
Any missing / malformed field falls back to the default. Safe to
|
| 122 |
+
call with ``None`` or ``{}`` β returns :func:`default_facets` in
|
| 123 |
+
those cases. Never raises; a broken facets block must not break a
|
| 124 |
+
call.
|
| 125 |
+
"""
|
| 126 |
+
d = default_facets()
|
| 127 |
+
if not isinstance(manifest, dict):
|
| 128 |
+
return d
|
| 129 |
+
raw = manifest.get("voice_facets")
|
| 130 |
+
if not isinstance(raw, dict):
|
| 131 |
+
return d
|
| 132 |
+
|
| 133 |
+
sig = raw.get("signature_tokens") or {}
|
| 134 |
+
if not isinstance(sig, dict):
|
| 135 |
+
sig = {}
|
| 136 |
+
|
| 137 |
+
reply_ns = raw.get("typical_reply_sentences")
|
| 138 |
+
if isinstance(reply_ns, list) and reply_ns:
|
| 139 |
+
reply_ns = [int(n) for n in reply_ns if isinstance(n, (int, float))]
|
| 140 |
+
else:
|
| 141 |
+
reply_ns = list(d.typical_reply_sentences)
|
| 142 |
+
|
| 143 |
+
return VoiceFacets(
|
| 144 |
+
warmth=_clamp(raw.get("warmth", d.warmth), 0.0, 1.0),
|
| 145 |
+
pace=_clamp(raw.get("pace", d.pace), 0.25, 2.0),
|
| 146 |
+
formality=_clamp(raw.get("formality", d.formality), 0.0, 1.0),
|
| 147 |
+
humor=_clamp(raw.get("humor", d.humor), 0.0, 1.0),
|
| 148 |
+
self_disclosure=_clamp(
|
| 149 |
+
raw.get("self_disclosure", d.self_disclosure), 0.0, 1.0
|
| 150 |
+
),
|
| 151 |
+
sentence_max_words=int(raw.get("sentence_max_words", d.sentence_max_words)),
|
| 152 |
+
typical_reply_sentences=reply_ns,
|
| 153 |
+
language_register=str(raw.get("language_register", d.language_register)),
|
| 154 |
+
ack_tokens=_tokens(sig.get("acknowledgments"), d.ack_tokens),
|
| 155 |
+
thinking_tokens=_tokens(sig.get("thinking"), d.thinking_tokens),
|
| 156 |
+
pre_closing_tokens=_tokens(sig.get("pre_closing"), d.pre_closing_tokens),
|
| 157 |
+
closing_tokens=_tokens(sig.get("closings"), d.closing_tokens),
|
| 158 |
+
backchannel_rate=_clamp(raw.get("backchannel_rate", 1.0), 0.0, 2.0),
|
| 159 |
+
)
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# ββ Runtime lookup β never mutates persona storage βββββββββββββββββββ
|
| 163 |
+
|
| 164 |
+
def for_persona_id(persona_id: Optional[str]) -> VoiceFacets:
|
| 165 |
+
"""Load facets for a persona without touching persona storage.
|
| 166 |
+
|
| 167 |
+
Looks up the persona's manifest (if any) via the existing personas
|
| 168 |
+
module. Any error returns the neutral default β persona_call must
|
| 169 |
+
degrade silently when persona storage has not yet been initialized
|
| 170 |
+
(e.g. early tests).
|
| 171 |
+
"""
|
| 172 |
+
if not persona_id:
|
| 173 |
+
return default_facets()
|
| 174 |
+
try:
|
| 175 |
+
# Lazy import to avoid any import-time coupling to personas.
|
| 176 |
+
from ..personas.repo import get_persona # type: ignore
|
| 177 |
+
manifest = get_persona(persona_id)
|
| 178 |
+
except Exception:
|
| 179 |
+
return default_facets()
|
| 180 |
+
return from_manifest(manifest)
|
backend/app/persona_call/latency.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pre-emptive filler scheduler β "hmm, one secβ¦" within the 700 ms
|
| 3 |
+
Stivers trouble threshold.
|
| 4 |
+
|
| 5 |
+
Because the MVP chat path is *non-streaming* (the compat endpoint
|
| 6 |
+
returns 501 on stream=true), the only way to cover long LLM latency
|
| 7 |
+
is to emit a server-originated filler BEFORE the real reply arrives.
|
| 8 |
+
|
| 9 |
+
Usage (from ws.py inside a turn)::
|
| 10 |
+
|
| 11 |
+
async with FillerScheduler(ws=..., facets=facets, cfg=cfg,
|
| 12 |
+
session_id=sid) as sched:
|
| 13 |
+
reply = await turn.run_turn(...)
|
| 14 |
+
# If the turn took longer than cfg.filler_emit_after_ms, the
|
| 15 |
+
# scheduler already emitted one 'assistant.filler' event to the
|
| 16 |
+
# client. Either way, the real reply follows.
|
| 17 |
+
|
| 18 |
+
Per the design doc: filler tokens are persona-specific with a
|
| 19 |
+
sensible universal fallback (covered by ``VoiceFacets.thinking_tokens``,
|
| 20 |
+
whose default pool is ["hmm", "let me think", "one sec", "hold on"]).
|
| 21 |
+
"""
|
| 22 |
+
from __future__ import annotations
|
| 23 |
+
|
| 24 |
+
import asyncio
|
| 25 |
+
import random
|
| 26 |
+
import time
|
| 27 |
+
from typing import Any, Callable, Dict, List, Optional
|
| 28 |
+
|
| 29 |
+
from .config import PersonaCallConfig
|
| 30 |
+
from .facets import VoiceFacets
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _now_ms() -> int:
|
| 34 |
+
return int(time.time() * 1000)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class FillerScheduler:
|
| 38 |
+
"""Async context-manager that emits at most ONE filler event per
|
| 39 |
+
turn, ``filler_emit_after_ms`` after entering the context. Cancel
|
| 40 |
+
on exit β if the LLM returned faster, no event is sent.
|
| 41 |
+
|
| 42 |
+
The event is emitted via the ``send`` callable injected at
|
| 43 |
+
construction, not by calling ``ws.send_text`` directly, so this
|
| 44 |
+
module stays free of fastapi imports and is trivially unit-testable.
|
| 45 |
+
"""
|
| 46 |
+
|
| 47 |
+
def __init__(
|
| 48 |
+
self,
|
| 49 |
+
*,
|
| 50 |
+
send: Callable[[Dict[str, Any]], "asyncio.Future"],
|
| 51 |
+
facets: VoiceFacets,
|
| 52 |
+
cfg: PersonaCallConfig,
|
| 53 |
+
session_id: str,
|
| 54 |
+
recent_thinking: Optional[List[str]] = None,
|
| 55 |
+
) -> None:
|
| 56 |
+
self._send = send
|
| 57 |
+
self._facets = facets
|
| 58 |
+
self._cfg = cfg
|
| 59 |
+
self._sid = session_id
|
| 60 |
+
self._recent = list(recent_thinking or [])
|
| 61 |
+
self._task: Optional[asyncio.Task] = None
|
| 62 |
+
self.emitted: bool = False
|
| 63 |
+
self.emitted_token: Optional[str] = None
|
| 64 |
+
|
| 65 |
+
def _pick_token(self) -> Optional[str]:
|
| 66 |
+
pool = [t for t in self._facets.thinking_tokens if t]
|
| 67 |
+
if not pool:
|
| 68 |
+
return None
|
| 69 |
+
last = (self._recent[-1].lower() if self._recent else "")
|
| 70 |
+
candidates = [t for t in pool if t.lower() != last] or pool
|
| 71 |
+
return random.choice(candidates)
|
| 72 |
+
|
| 73 |
+
async def _fire(self) -> None:
|
| 74 |
+
delay_ms = max(0, int(self._cfg.filler_emit_after_ms))
|
| 75 |
+
try:
|
| 76 |
+
await asyncio.sleep(delay_ms / 1000.0)
|
| 77 |
+
except asyncio.CancelledError:
|
| 78 |
+
return
|
| 79 |
+
token = self._pick_token()
|
| 80 |
+
if not token:
|
| 81 |
+
return
|
| 82 |
+
try:
|
| 83 |
+
await self._send({
|
| 84 |
+
"type": "assistant.filler",
|
| 85 |
+
"payload": {
|
| 86 |
+
"token": token,
|
| 87 |
+
"ts": _now_ms(),
|
| 88 |
+
"session_id": self._sid,
|
| 89 |
+
},
|
| 90 |
+
})
|
| 91 |
+
self.emitted = True
|
| 92 |
+
self.emitted_token = token
|
| 93 |
+
except Exception:
|
| 94 |
+
# Best-effort β never let a filler failure break the turn.
|
| 95 |
+
pass
|
| 96 |
+
|
| 97 |
+
async def __aenter__(self) -> "FillerScheduler":
|
| 98 |
+
self._task = asyncio.create_task(self._fire())
|
| 99 |
+
return self
|
| 100 |
+
|
| 101 |
+
async def __aexit__(self, exc_type, exc, tb) -> None:
|
| 102 |
+
if self._task is not None:
|
| 103 |
+
self._task.cancel()
|
| 104 |
+
try:
|
| 105 |
+
await self._task
|
| 106 |
+
except Exception:
|
| 107 |
+
pass
|
backend/app/persona_call/openings.py
ADDED
|
@@ -0,0 +1,334 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
persona_call.openings β "theory of answering" template bank.
|
| 3 |
+
|
| 4 |
+
Adds the deterministic first-turn answer the persona uses when a call
|
| 5 |
+
connects. Backed by conversation-analysis research on phone openings
|
| 6 |
+
(Schegloff 1968, Hopper 1992) and cross-cultural data (Liddicoat
|
| 7 |
+
2021, ch. 9); augmented with call-center industry practice for the
|
| 8 |
+
formal-register templates.
|
| 9 |
+
|
| 10 |
+
Why a template bank instead of "let the LLM improvise"
|
| 11 |
+
------------------------------------------------------
|
| 12 |
+
Phone openings are the single most ritualized slot in human
|
| 13 |
+
conversation β every culture has a narrow, low-entropy answer
|
| 14 |
+
inventory. A free-form LLM greeting is both slower (latency hurts
|
| 15 |
+
most on turn 1) and less natural (LLMs over-generate: "Hello there!
|
| 16 |
+
What a wonderful day to connect with youβ¦"). A curated bank of
|
| 17 |
+
short templates fires in <5 ms, is persona-aware, is rotated so the
|
| 18 |
+
same caller never hears the same greeting twice in a row, and is
|
| 19 |
+
trivially auditable.
|
| 20 |
+
|
| 21 |
+
Design constraints
|
| 22 |
+
------------------
|
| 23 |
+
1. **Additive.** This module is new; nothing else changes. The
|
| 24 |
+
existing ``directive.compose()`` uses it opportunistically in
|
| 25 |
+
the opening phase only.
|
| 26 |
+
2. **Persona-preserving.** A template is a *seed*, not a mandate:
|
| 27 |
+
the chosen text flows through the persona's own TTS voice and
|
| 28 |
+
the LLM never sees it as an instruction to copy. If a persona
|
| 29 |
+
ships ``voice_facets.signature_tokens.openings``, that override
|
| 30 |
+
wins; templates from this module are used only when the persona
|
| 31 |
+
declines to ship its own.
|
| 32 |
+
3. **Deterministic under tests.** Selection is seeded by
|
| 33 |
+
(session_id, turn_index) so the same call always picks the same
|
| 34 |
+
template β regression tests don't flake.
|
| 35 |
+
4. **Rotated.** A separate persisted ledger
|
| 36 |
+
(``persona_call_state.recent_openings``) already tracks the
|
| 37 |
+
last few opener tokens the persona used; we extend it to cover
|
| 38 |
+
full template IDs so cross-call repetition is avoided too.
|
| 39 |
+
|
| 40 |
+
Template families (drawn from Hopper 1992 + call-center practice)
|
| 41 |
+
-----------------------------------------------------------------
|
| 42 |
+
- **summons_answer** β the minimalist acknowledge-you've-been-summoned
|
| 43 |
+
move: "Hello?", "Yes?", "Mm?" (Schegloff 1968, turn 1).
|
| 44 |
+
- **identification** β name-forward answer, the default for most
|
| 45 |
+
non-intimate contexts: "Atlas speaking.", "This is Atlas."
|
| 46 |
+
- **greeting_plus_name** β warmer, the default personal-call opener
|
| 47 |
+
in English: "Hi, Atlas here.", "Hey β Atlas."
|
| 48 |
+
- **time_of_day** β morning/evening variants: "Good evening, Atlas.",
|
| 49 |
+
"Morning β Atlas.".
|
| 50 |
+
- **intimate_reopen** β for repeat callers (weeks_since_last_call
|
| 51 |
+
< 2): "Hey you.", "Oh hi!", "Good to hear from you.".
|
| 52 |
+
- **call_center_formal** β for personas with formality β₯ 0.75:
|
| 53 |
+
"Thank you for calling. This is Atlas, how can I help?" and
|
| 54 |
+
variants. The *only* family that includes a how-may-I-help tail.
|
| 55 |
+
- **late_night_brief** β after 22:00 local: "Heyβ¦", "Mm, yeah?",
|
| 56 |
+
"Yeah β hi.". Short on purpose.
|
| 57 |
+
|
| 58 |
+
Each template carries a ``precond`` callable that decides whether it
|
| 59 |
+
fits the current ``(facets, env, state)``. Selection is a two-pass
|
| 60 |
+
filter+weight: first drop ineligible templates, then weight the
|
| 61 |
+
survivors by persona fit.
|
| 62 |
+
"""
|
| 63 |
+
|
| 64 |
+
from __future__ import annotations
|
| 65 |
+
|
| 66 |
+
import hashlib
|
| 67 |
+
import random
|
| 68 |
+
from dataclasses import dataclass, field
|
| 69 |
+
from typing import Callable, Iterable, List, Optional
|
| 70 |
+
|
| 71 |
+
from .context import CallEnv
|
| 72 |
+
from .facets import VoiceFacets
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
# ββ Template record ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 76 |
+
|
| 77 |
+
@dataclass(frozen=True)
|
| 78 |
+
class OpeningTemplate:
|
| 79 |
+
"""One row in the template bank.
|
| 80 |
+
|
| 81 |
+
Attributes
|
| 82 |
+
----------
|
| 83 |
+
id
|
| 84 |
+
Stable identifier persisted in the anti-repetition ledger.
|
| 85 |
+
Short snake_case.
|
| 86 |
+
family
|
| 87 |
+
One of the seven families in the module docstring. Used by
|
| 88 |
+
selection weighting + tests.
|
| 89 |
+
text
|
| 90 |
+
The template string. ``{name}`` is interpolated with the
|
| 91 |
+
persona's display name (or swallowed if empty). No other
|
| 92 |
+
interpolation β keeping templates 100% literal means a
|
| 93 |
+
reviewer can read the bank and know exactly what can ship.
|
| 94 |
+
formality_range
|
| 95 |
+
Inclusive ``(lo, hi)`` on the persona's formality facet.
|
| 96 |
+
Templates outside the persona's range are filtered out.
|
| 97 |
+
precond
|
| 98 |
+
Final predicate run against ``(facets, env)``. Returns True
|
| 99 |
+
iff the template is eligible *right now*. Typical uses:
|
| 100 |
+
late-night variants only after 22:00, intimate variants only
|
| 101 |
+
for recurring callers.
|
| 102 |
+
"""
|
| 103 |
+
id: str
|
| 104 |
+
family: str
|
| 105 |
+
text: str
|
| 106 |
+
formality_range: tuple = (0.0, 1.0)
|
| 107 |
+
precond: Callable[[VoiceFacets, CallEnv], bool] = field(
|
| 108 |
+
default=lambda _f, _e: True,
|
| 109 |
+
)
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
# ββ Precondition helpers β readable and reusable. ββββββββββββββββββββ
|
| 113 |
+
|
| 114 |
+
def _any_time(_f: VoiceFacets, _e: CallEnv) -> bool:
|
| 115 |
+
return True
|
| 116 |
+
|
| 117 |
+
def _recurring(_f: VoiceFacets, env: CallEnv) -> bool:
|
| 118 |
+
"""Fire only when this caller has spoken to this persona recently.
|
| 119 |
+
Guards against a cold-call receiving the intimate family."""
|
| 120 |
+
return 0 <= env.weeks_since_last_call < 2
|
| 121 |
+
|
| 122 |
+
def _late_night(_f: VoiceFacets, env: CallEnv) -> bool:
|
| 123 |
+
return env.local_hour >= 22 or env.local_hour < 6
|
| 124 |
+
|
| 125 |
+
def _morning(_f: VoiceFacets, env: CallEnv) -> bool:
|
| 126 |
+
return 5 <= env.local_hour < 11
|
| 127 |
+
|
| 128 |
+
def _afternoon(_f: VoiceFacets, env: CallEnv) -> bool:
|
| 129 |
+
return 11 <= env.local_hour < 17
|
| 130 |
+
|
| 131 |
+
def _evening(_f: VoiceFacets, env: CallEnv) -> bool:
|
| 132 |
+
return 17 <= env.local_hour < 22
|
| 133 |
+
|
| 134 |
+
|
| 135 |
+
# ββ The bank βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 136 |
+
# Deliberately small β 23 templates across 7 families. Adding more is
|
| 137 |
+
# cheap but every addition increases the risk of one slipping through
|
| 138 |
+
# review in a voice that doesn't fit the persona. Keep the bar high.
|
| 139 |
+
|
| 140 |
+
OPENING_BANK: tuple = (
|
| 141 |
+
# 1. summons_answer β the Schegloff minimum. Any formality.
|
| 142 |
+
OpeningTemplate("sa_hello", "summons_answer", "Hello?"),
|
| 143 |
+
OpeningTemplate("sa_hello_open", "summons_answer", "Hello β who's this?"),
|
| 144 |
+
OpeningTemplate("sa_yes", "summons_answer", "Yes?",
|
| 145 |
+
formality_range=(0.0, 0.6)),
|
| 146 |
+
OpeningTemplate("sa_yeah", "summons_answer", "Yeah?",
|
| 147 |
+
formality_range=(0.0, 0.45)),
|
| 148 |
+
|
| 149 |
+
# 2. identification β name-forward. Default for 0.3 β€ formality β€ 0.8.
|
| 150 |
+
OpeningTemplate("id_name_speaking", "identification",
|
| 151 |
+
"{name} speaking.",
|
| 152 |
+
formality_range=(0.4, 1.0)),
|
| 153 |
+
OpeningTemplate("id_this_is", "identification",
|
| 154 |
+
"This is {name}.",
|
| 155 |
+
formality_range=(0.35, 1.0)),
|
| 156 |
+
OpeningTemplate("id_hello_name", "identification",
|
| 157 |
+
"Hello, this is {name}.",
|
| 158 |
+
formality_range=(0.4, 1.0)),
|
| 159 |
+
|
| 160 |
+
# 3. greeting_plus_name β warmer, casual. Default personal-call.
|
| 161 |
+
OpeningTemplate("gn_hi_here", "greeting_plus_name",
|
| 162 |
+
"Hi, {name} here.",
|
| 163 |
+
formality_range=(0.2, 0.75)),
|
| 164 |
+
OpeningTemplate("gn_hey_name", "greeting_plus_name",
|
| 165 |
+
"Hey β {name}.",
|
| 166 |
+
formality_range=(0.0, 0.6)),
|
| 167 |
+
OpeningTemplate("gn_hi_hi", "greeting_plus_name",
|
| 168 |
+
"Hi hi.",
|
| 169 |
+
formality_range=(0.0, 0.55)),
|
| 170 |
+
|
| 171 |
+
# 4. time_of_day β used only at matching hour, amortised across
|
| 172 |
+
# the bank so it never dominates.
|
| 173 |
+
OpeningTemplate("td_morning", "time_of_day",
|
| 174 |
+
"Morning β {name}.",
|
| 175 |
+
precond=_morning),
|
| 176 |
+
OpeningTemplate("td_good_morning", "time_of_day",
|
| 177 |
+
"Good morning, {name} speaking.",
|
| 178 |
+
formality_range=(0.5, 1.0), precond=_morning),
|
| 179 |
+
OpeningTemplate("td_afternoon", "time_of_day",
|
| 180 |
+
"Good afternoon β {name}.",
|
| 181 |
+
formality_range=(0.4, 1.0), precond=_afternoon),
|
| 182 |
+
OpeningTemplate("td_evening", "time_of_day",
|
| 183 |
+
"Good evening.",
|
| 184 |
+
formality_range=(0.4, 1.0), precond=_evening),
|
| 185 |
+
|
| 186 |
+
# 5. intimate_reopen β recurring callers only.
|
| 187 |
+
OpeningTemplate("int_hey_you", "intimate_reopen",
|
| 188 |
+
"Hey you.",
|
| 189 |
+
formality_range=(0.0, 0.5), precond=_recurring),
|
| 190 |
+
OpeningTemplate("int_oh_hi", "intimate_reopen",
|
| 191 |
+
"Oh, hi!",
|
| 192 |
+
formality_range=(0.0, 0.6), precond=_recurring),
|
| 193 |
+
OpeningTemplate("int_good_to_hear", "intimate_reopen",
|
| 194 |
+
"Hey β good to hear from you.",
|
| 195 |
+
formality_range=(0.15, 0.7), precond=_recurring),
|
| 196 |
+
|
| 197 |
+
# 6. call_center_formal β high-formality only; the only family
|
| 198 |
+
# that volunteers help.
|
| 199 |
+
OpeningTemplate("cc_thank_you_calling", "call_center_formal",
|
| 200 |
+
"Thank you for calling. This is {name}, how can I help?",
|
| 201 |
+
formality_range=(0.75, 1.0)),
|
| 202 |
+
OpeningTemplate("cc_name_how_help", "call_center_formal",
|
| 203 |
+
"{name} speaking β how can I help?",
|
| 204 |
+
formality_range=(0.7, 1.0)),
|
| 205 |
+
OpeningTemplate("cc_standing_by", "call_center_formal",
|
| 206 |
+
"Hello, {name} here β what can I do for you?",
|
| 207 |
+
formality_range=(0.65, 1.0)),
|
| 208 |
+
|
| 209 |
+
# 7. late_night_brief β just picked up the phone, mid-yawn.
|
| 210 |
+
OpeningTemplate("ln_hey", "late_night_brief",
|
| 211 |
+
"Heyβ¦",
|
| 212 |
+
formality_range=(0.0, 0.55), precond=_late_night),
|
| 213 |
+
OpeningTemplate("ln_yeah_hi", "late_night_brief",
|
| 214 |
+
"Yeah β hi.",
|
| 215 |
+
formality_range=(0.0, 0.5), precond=_late_night),
|
| 216 |
+
OpeningTemplate("ln_mm_yeah", "late_night_brief",
|
| 217 |
+
"Mm, yeah?",
|
| 218 |
+
formality_range=(0.0, 0.45), precond=_late_night),
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
# ββ Public API βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 223 |
+
|
| 224 |
+
def _interpolate(tpl: str, name: str) -> str:
|
| 225 |
+
"""Render ``{name}`` in the template. If the caller has no usable
|
| 226 |
+
display name, swallow the placeholder cleanly so we never ship
|
| 227 |
+
literal ``{name}``."""
|
| 228 |
+
safe = (name or "").strip()
|
| 229 |
+
if not safe:
|
| 230 |
+
# Drop the placeholder + surrounding whitespace around it so
|
| 231 |
+
# "This is {name}." β "This is." wouldn't be ugly. We replace
|
| 232 |
+
# with a near-natural collapse.
|
| 233 |
+
return (
|
| 234 |
+
tpl.replace(", this is {name}.", ".")
|
| 235 |
+
.replace(" this is {name}.", ".")
|
| 236 |
+
.replace(" β {name}.", ".")
|
| 237 |
+
.replace("{name} speaking.", "Speaking.")
|
| 238 |
+
.replace("{name} here.", "Here.")
|
| 239 |
+
.replace("{name}", "")
|
| 240 |
+
.strip()
|
| 241 |
+
)
|
| 242 |
+
return tpl.replace("{name}", safe)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def eligible_templates(
|
| 246 |
+
facets: VoiceFacets,
|
| 247 |
+
env: CallEnv,
|
| 248 |
+
*,
|
| 249 |
+
bank: Iterable[OpeningTemplate] = OPENING_BANK,
|
| 250 |
+
forbidden_ids: Iterable[str] = (),
|
| 251 |
+
) -> List[OpeningTemplate]:
|
| 252 |
+
"""Filter the bank to what fits the persona + environment + ledger.
|
| 253 |
+
|
| 254 |
+
Two-pass filter:
|
| 255 |
+
1. Formality range contains ``facets.formality``.
|
| 256 |
+
2. ``precond(facets, env)`` returns True.
|
| 257 |
+
3. ``id`` not in ``forbidden_ids`` (the anti-repetition ledger).
|
| 258 |
+
"""
|
| 259 |
+
banned = set(forbidden_ids or ())
|
| 260 |
+
out: List[OpeningTemplate] = []
|
| 261 |
+
for t in bank:
|
| 262 |
+
lo, hi = t.formality_range
|
| 263 |
+
if not (lo <= facets.formality <= hi):
|
| 264 |
+
continue
|
| 265 |
+
if not t.precond(facets, env):
|
| 266 |
+
continue
|
| 267 |
+
if t.id in banned:
|
| 268 |
+
continue
|
| 269 |
+
out.append(t)
|
| 270 |
+
return out
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
def _seed(session_id: str, turn_index: int) -> int:
|
| 274 |
+
"""Stable seed so tests don't flake.
|
| 275 |
+
|
| 276 |
+
Uses blake2b rather than hash() because CPython randomizes
|
| 277 |
+
``hash()`` between processes (PYTHONHASHSEED), which makes the
|
| 278 |
+
selected template drift under test isolation and is a subtle
|
| 279 |
+
source of flakes.
|
| 280 |
+
"""
|
| 281 |
+
h = hashlib.blake2b(
|
| 282 |
+
f"{session_id}:{turn_index}".encode("utf-8"), digest_size=8,
|
| 283 |
+
)
|
| 284 |
+
return int.from_bytes(h.digest(), "big")
|
| 285 |
+
|
| 286 |
+
|
| 287 |
+
def choose(
|
| 288 |
+
facets: VoiceFacets,
|
| 289 |
+
env: CallEnv,
|
| 290 |
+
*,
|
| 291 |
+
session_id: str,
|
| 292 |
+
turn_index: int = 1,
|
| 293 |
+
forbidden_ids: Iterable[str] = (),
|
| 294 |
+
bank: Iterable[OpeningTemplate] = OPENING_BANK,
|
| 295 |
+
) -> Optional[OpeningTemplate]:
|
| 296 |
+
"""Pick one eligible template.
|
| 297 |
+
|
| 298 |
+
Returns the persona's own signature opener when ``facets`` ships a
|
| 299 |
+
custom pool β otherwise picks from the filtered bank. Never
|
| 300 |
+
returns an ``OpeningTemplate`` that matches a forbidden id.
|
| 301 |
+
|
| 302 |
+
If no eligible template survives the filter (rare β would require
|
| 303 |
+
an unusual formality + time-of-day combination and a ledger full
|
| 304 |
+
of every normally-eligible id), this returns ``None`` and the
|
| 305 |
+
caller should fall back to ``"Hello?"``, which is why the
|
| 306 |
+
summons_answer family has no preconditions.
|
| 307 |
+
"""
|
| 308 |
+
pool = eligible_templates(facets, env, bank=bank, forbidden_ids=forbidden_ids)
|
| 309 |
+
if not pool:
|
| 310 |
+
return None
|
| 311 |
+
rnd = random.Random(_seed(session_id, turn_index))
|
| 312 |
+
return rnd.choice(pool)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def render(
|
| 316 |
+
facets: VoiceFacets,
|
| 317 |
+
env: CallEnv,
|
| 318 |
+
*,
|
| 319 |
+
persona_display_name: str = "",
|
| 320 |
+
session_id: str,
|
| 321 |
+
turn_index: int = 1,
|
| 322 |
+
forbidden_ids: Iterable[str] = (),
|
| 323 |
+
) -> Optional[str]:
|
| 324 |
+
"""End-to-end helper: pick + interpolate. Returns None on empty
|
| 325 |
+
pool so the caller can fall back safely."""
|
| 326 |
+
tpl = choose(
|
| 327 |
+
facets, env,
|
| 328 |
+
session_id=session_id,
|
| 329 |
+
turn_index=turn_index,
|
| 330 |
+
forbidden_ids=forbidden_ids,
|
| 331 |
+
)
|
| 332 |
+
if tpl is None:
|
| 333 |
+
return None
|
| 334 |
+
return _interpolate(tpl.text, persona_display_name)
|
backend/app/persona_call/repeat.py
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Anti-repetition ledger.
|
| 3 |
+
|
| 4 |
+
Three windowed deques per session, all backed by the JSON columns on
|
| 5 |
+
``persona_call_state``:
|
| 6 |
+
|
| 7 |
+
recent_acks last N acknowledgment tokens the persona used
|
| 8 |
+
recent_openers last N first-words of the persona's turns
|
| 9 |
+
recent_closings last N pre-closing tokens
|
| 10 |
+
|
| 11 |
+
Read API returns the set to FORBID in the next turn's directive; write
|
| 12 |
+
API is called after the reply comes back and extracts the new token via
|
| 13 |
+
a regex-first / LLM-never extractor so it stays cheap.
|
| 14 |
+
|
| 15 |
+
No LLM calls anywhere here β the extraction is regex + token-pool
|
| 16 |
+
membership. If the persona's reply starts with a token that isn't in
|
| 17 |
+
its own voice_facets pool, we store the first lowercased word anyway;
|
| 18 |
+
it still anchors the entropy metric.
|
| 19 |
+
"""
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import re
|
| 23 |
+
from typing import Dict, List, Optional, Set, Tuple
|
| 24 |
+
|
| 25 |
+
from .facets import VoiceFacets
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
_WORD_BOUNDARY = re.compile(r"^[\s\W]*([a-z][a-z\-']*)", re.I)
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _first_word(reply: str) -> Optional[str]:
|
| 32 |
+
if not reply:
|
| 33 |
+
return None
|
| 34 |
+
m = _WORD_BOUNDARY.match(reply.strip())
|
| 35 |
+
if not m:
|
| 36 |
+
return None
|
| 37 |
+
return m.group(1).lower()
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def _roll(deque: List[str], token: str, size: int) -> List[str]:
|
| 41 |
+
out = list(deque or [])
|
| 42 |
+
if token:
|
| 43 |
+
out.append(token)
|
| 44 |
+
if len(out) > size:
|
| 45 |
+
out = out[-size:]
|
| 46 |
+
return out
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# ββ READ: directive paragraph that forbids recent tokens βββββββββββββ
|
| 50 |
+
|
| 51 |
+
def forbid_clause(
|
| 52 |
+
state: Dict[str, object],
|
| 53 |
+
facets: VoiceFacets,
|
| 54 |
+
*,
|
| 55 |
+
max_len: int = 80,
|
| 56 |
+
) -> str:
|
| 57 |
+
"""Return the one-line directive paragraph about recent tokens.
|
| 58 |
+
|
| 59 |
+
Empty string when there's nothing to forbid yet β keeps the
|
| 60 |
+
per-turn suffix short on early turns."""
|
| 61 |
+
acks = state.get("recent_acks") or []
|
| 62 |
+
openers = state.get("recent_openers") or []
|
| 63 |
+
if not acks and not openers:
|
| 64 |
+
return ""
|
| 65 |
+
|
| 66 |
+
dropped: Set[str] = set()
|
| 67 |
+
for t in acks:
|
| 68 |
+
if isinstance(t, str) and t:
|
| 69 |
+
dropped.add(t.lower())
|
| 70 |
+
for t in openers:
|
| 71 |
+
if isinstance(t, str) and t:
|
| 72 |
+
dropped.add(t.lower())
|
| 73 |
+
|
| 74 |
+
if not dropped:
|
| 75 |
+
return ""
|
| 76 |
+
|
| 77 |
+
pool = [t for t in facets.ack_tokens if t.lower() not in dropped]
|
| 78 |
+
fresh_hint = ""
|
| 79 |
+
if pool:
|
| 80 |
+
fresh_hint = f" Prefer a fresh one from: {', '.join(pool[:4])}."
|
| 81 |
+
|
| 82 |
+
forbidden = sorted(dropped)[: max_len // 5] # rough clamp
|
| 83 |
+
return (
|
| 84 |
+
"Avoid repeating your recent openers / acknowledgments: "
|
| 85 |
+
f"{', '.join(forbidden)}." + fresh_hint
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# ββ WRITE: record the persona's reply into the ledger ββββββββββββββββ
|
| 90 |
+
|
| 91 |
+
def record_reply(
|
| 92 |
+
state: Dict[str, object],
|
| 93 |
+
reply: str,
|
| 94 |
+
facets: VoiceFacets,
|
| 95 |
+
cfg_ack_window: int,
|
| 96 |
+
cfg_opener_window: int,
|
| 97 |
+
) -> Tuple[List[str], List[str]]:
|
| 98 |
+
"""Roll the ledger forward. Returns the new (recent_acks,
|
| 99 |
+
recent_openers) lists for ``store.update_state``."""
|
| 100 |
+
token = _first_word(reply) or ""
|
| 101 |
+
acks = state.get("recent_acks") or []
|
| 102 |
+
openers = state.get("recent_openers") or []
|
| 103 |
+
|
| 104 |
+
# An ack is anything that appears in the persona's ack pool AND
|
| 105 |
+
# appears in the first three words. That matches how human phone
|
| 106 |
+
# acks actually work ("mm yeah I hear you"): the ack is the front
|
| 107 |
+
# of the turn, not a whole separate utterance.
|
| 108 |
+
front = " ".join((reply or "").lower().strip().split()[:3])
|
| 109 |
+
matched_ack = None
|
| 110 |
+
for pool_token in facets.ack_tokens:
|
| 111 |
+
if pool_token in front:
|
| 112 |
+
matched_ack = pool_token
|
| 113 |
+
break
|
| 114 |
+
|
| 115 |
+
new_acks = _roll(list(acks), matched_ack or "", cfg_ack_window)
|
| 116 |
+
new_openers = _roll(list(openers), token, cfg_opener_window)
|
| 117 |
+
return new_acks, new_openers
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def record_closing(
|
| 121 |
+
state: Dict[str, object],
|
| 122 |
+
token: str,
|
| 123 |
+
window: int = 3,
|
| 124 |
+
) -> List[str]:
|
| 125 |
+
"""Roll the closings deque. Called when the closing state machine
|
| 126 |
+
chooses a pre-closing token."""
|
| 127 |
+
prev = state.get("recent_closings") or []
|
| 128 |
+
return _roll(list(prev), token.lower() if token else "", window)
|
backend/app/persona_call/router.py
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Public HTTP surface for persona_call.
|
| 3 |
+
|
| 4 |
+
Five endpoints, all minor β this module is primarily called from
|
| 5 |
+
``voice_call/ws.py`` at runtime. These REST routes are for
|
| 6 |
+
observability + admin tuning + client facets lookup.
|
| 7 |
+
|
| 8 |
+
Mount point in main.py is ONE feature-flagged try/except block.
|
| 9 |
+
With ``PERSONA_CALL_ENABLED=false`` :func:`build_router` returns
|
| 10 |
+
an empty router β every path 404s, zero schema is created.
|
| 11 |
+
"""
|
| 12 |
+
from __future__ import annotations
|
| 13 |
+
|
| 14 |
+
import json
|
| 15 |
+
from typing import Any, Dict, List, Optional
|
| 16 |
+
|
| 17 |
+
from fastapi import (
|
| 18 |
+
APIRouter, Cookie, Depends, Header, HTTPException, Query,
|
| 19 |
+
)
|
| 20 |
+
from pydantic import BaseModel
|
| 21 |
+
|
| 22 |
+
from . import facets as facets_mod
|
| 23 |
+
from . import store
|
| 24 |
+
from .config import PersonaCallConfig
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
# ββ auth passthrough (matches main._scoped_user_or_none) βββββββββββββ
|
| 28 |
+
|
| 29 |
+
def _resolve_user(
|
| 30 |
+
authorization: str,
|
| 31 |
+
homepilot_session: Optional[str],
|
| 32 |
+
) -> dict:
|
| 33 |
+
from ..users import (
|
| 34 |
+
ensure_users_tables, _validate_token,
|
| 35 |
+
get_or_create_default_user, count_users,
|
| 36 |
+
)
|
| 37 |
+
ensure_users_tables()
|
| 38 |
+
token = ""
|
| 39 |
+
if authorization and authorization.lower().startswith("bearer "):
|
| 40 |
+
token = authorization.split(" ", 1)[1].strip()
|
| 41 |
+
if not token and homepilot_session:
|
| 42 |
+
token = homepilot_session.strip()
|
| 43 |
+
user = _validate_token(token) if token else None
|
| 44 |
+
if user:
|
| 45 |
+
return user
|
| 46 |
+
if count_users() > 1:
|
| 47 |
+
raise HTTPException(status_code=401, detail="Authentication required")
|
| 48 |
+
return get_or_create_default_user()
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
# ββ response models ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 52 |
+
|
| 53 |
+
class FacetsResp(BaseModel):
|
| 54 |
+
persona_id: Optional[str]
|
| 55 |
+
warmth: float
|
| 56 |
+
pace: float
|
| 57 |
+
formality: float
|
| 58 |
+
humor: float
|
| 59 |
+
self_disclosure: float
|
| 60 |
+
sentence_max_words: int
|
| 61 |
+
typical_reply_sentences: List[int]
|
| 62 |
+
language_register: str
|
| 63 |
+
signature_tokens: Dict[str, List[str]]
|
| 64 |
+
backchannel_rate: float
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
class StateResp(BaseModel):
|
| 68 |
+
session_id: str
|
| 69 |
+
phase: str
|
| 70 |
+
turn_index: int
|
| 71 |
+
skipped_how_are_you: bool
|
| 72 |
+
asked_reason_fallback: bool
|
| 73 |
+
reason_for_call: str
|
| 74 |
+
pre_closing_trigger: Optional[str]
|
| 75 |
+
recent_acks: List[str]
|
| 76 |
+
recent_openers: List[str]
|
| 77 |
+
recent_closings: List[str]
|
| 78 |
+
caller_context: Dict[str, Any]
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
class DirectiveResp(BaseModel):
|
| 82 |
+
session_id: str
|
| 83 |
+
turn_index: int
|
| 84 |
+
phase: str
|
| 85 |
+
applied: bool
|
| 86 |
+
system_suffix: str
|
| 87 |
+
post_directives: List[str]
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ββ router builder βββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 91 |
+
|
| 92 |
+
def build_router(cfg: PersonaCallConfig) -> APIRouter:
|
| 93 |
+
"""Construct the FastAPI router. Empty if ``cfg.enabled`` is
|
| 94 |
+
False so main.py's mount is always a safe no-op."""
|
| 95 |
+
r = APIRouter(prefix="/v1/persona-call", tags=["persona-call"])
|
| 96 |
+
if not cfg.enabled:
|
| 97 |
+
return r
|
| 98 |
+
|
| 99 |
+
@r.get("/facets/{persona_id}", response_model=FacetsResp)
|
| 100 |
+
def get_facets(persona_id: str):
|
| 101 |
+
f = facets_mod.for_persona_id(persona_id)
|
| 102 |
+
return FacetsResp(
|
| 103 |
+
persona_id=persona_id,
|
| 104 |
+
warmth=f.warmth,
|
| 105 |
+
pace=f.pace,
|
| 106 |
+
formality=f.formality,
|
| 107 |
+
humor=f.humor,
|
| 108 |
+
self_disclosure=f.self_disclosure,
|
| 109 |
+
sentence_max_words=f.sentence_max_words,
|
| 110 |
+
typical_reply_sentences=f.typical_reply_sentences,
|
| 111 |
+
language_register=f.language_register,
|
| 112 |
+
signature_tokens={
|
| 113 |
+
"acknowledgments": list(f.ack_tokens),
|
| 114 |
+
"thinking": list(f.thinking_tokens),
|
| 115 |
+
"pre_closing": list(f.pre_closing_tokens),
|
| 116 |
+
"closings": list(f.closing_tokens),
|
| 117 |
+
},
|
| 118 |
+
backchannel_rate=f.backchannel_rate,
|
| 119 |
+
)
|
| 120 |
+
|
| 121 |
+
@r.get("/state/{session_id}", response_model=StateResp)
|
| 122 |
+
def get_state(
|
| 123 |
+
session_id: str,
|
| 124 |
+
authorization: str = Header(default=""),
|
| 125 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 126 |
+
):
|
| 127 |
+
"""Runtime state for the session. Owner-bound via the
|
| 128 |
+
underlying voice_call_sessions row β if the session doesn't
|
| 129 |
+
belong to the caller, return 404 (probe-safe)."""
|
| 130 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 131 |
+
# Ownership is enforced by looking up the voice_call_sessions
|
| 132 |
+
# row first. If that is missing or foreign, we 404 before
|
| 133 |
+
# touching persona_call_state at all.
|
| 134 |
+
try:
|
| 135 |
+
from ..voice_call.store import get_session_for_owner
|
| 136 |
+
vc_row = get_session_for_owner(session_id, user["id"])
|
| 137 |
+
except Exception:
|
| 138 |
+
vc_row = None
|
| 139 |
+
if vc_row is None:
|
| 140 |
+
raise HTTPException(status_code=404, detail="Session not found")
|
| 141 |
+
s = store.get_state(session_id)
|
| 142 |
+
if s is None:
|
| 143 |
+
# Session exists but no persona_call state yet β return a
|
| 144 |
+
# zero-ed row rather than 404 so clients can poll safely
|
| 145 |
+
# before the first turn.
|
| 146 |
+
return StateResp(
|
| 147 |
+
session_id=session_id,
|
| 148 |
+
phase="opening",
|
| 149 |
+
turn_index=0,
|
| 150 |
+
skipped_how_are_you=False,
|
| 151 |
+
asked_reason_fallback=False,
|
| 152 |
+
reason_for_call="",
|
| 153 |
+
pre_closing_trigger=None,
|
| 154 |
+
recent_acks=[],
|
| 155 |
+
recent_openers=[],
|
| 156 |
+
recent_closings=[],
|
| 157 |
+
caller_context={},
|
| 158 |
+
)
|
| 159 |
+
return StateResp(
|
| 160 |
+
session_id=s["session_id"],
|
| 161 |
+
phase=str(s.get("phase") or "opening"),
|
| 162 |
+
turn_index=int(s.get("turn_index") or 0),
|
| 163 |
+
skipped_how_are_you=bool(s.get("skipped_how_are_you")),
|
| 164 |
+
asked_reason_fallback=bool(s.get("asked_reason_fallback")),
|
| 165 |
+
reason_for_call=str(s.get("reason_for_call") or ""),
|
| 166 |
+
pre_closing_trigger=s.get("pre_closing_trigger"),
|
| 167 |
+
recent_acks=list(s.get("recent_acks") or []),
|
| 168 |
+
recent_openers=list(s.get("recent_openers") or []),
|
| 169 |
+
recent_closings=list(s.get("recent_closings") or []),
|
| 170 |
+
caller_context=dict(s.get("caller_context") or {}),
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
@r.get("/last-directive/{session_id}", response_model=DirectiveResp)
|
| 174 |
+
def get_last_directive(
|
| 175 |
+
session_id: str,
|
| 176 |
+
authorization: str = Header(default=""),
|
| 177 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 178 |
+
):
|
| 179 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 180 |
+
try:
|
| 181 |
+
from ..voice_call.store import get_session_for_owner
|
| 182 |
+
vc_row = get_session_for_owner(session_id, user["id"])
|
| 183 |
+
except Exception:
|
| 184 |
+
vc_row = None
|
| 185 |
+
if vc_row is None:
|
| 186 |
+
raise HTTPException(status_code=404, detail="Session not found")
|
| 187 |
+
d = store.last_directive(session_id)
|
| 188 |
+
if d is None:
|
| 189 |
+
raise HTTPException(status_code=404, detail="No directive yet")
|
| 190 |
+
return DirectiveResp(
|
| 191 |
+
session_id=d["session_id"],
|
| 192 |
+
turn_index=int(d["turn_index"]),
|
| 193 |
+
phase=str(d["phase"]),
|
| 194 |
+
applied=bool(d["applied"]),
|
| 195 |
+
system_suffix=str(d["system_suffix"] or ""),
|
| 196 |
+
post_directives=list(d["post_directives"] or []),
|
| 197 |
+
)
|
| 198 |
+
|
| 199 |
+
@r.post("/reset/{session_id}")
|
| 200 |
+
def reset_session(
|
| 201 |
+
session_id: str,
|
| 202 |
+
authorization: str = Header(default=""),
|
| 203 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 204 |
+
):
|
| 205 |
+
"""Wipe the anti-repetition ledgers for a session. Owner-bound.
|
| 206 |
+
Used by QA + tests; harmless in production because the phase
|
| 207 |
+
machine just starts fresh from 'opening'."""
|
| 208 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 209 |
+
try:
|
| 210 |
+
from ..voice_call.store import get_session_for_owner
|
| 211 |
+
vc_row = get_session_for_owner(session_id, user["id"])
|
| 212 |
+
except Exception:
|
| 213 |
+
vc_row = None
|
| 214 |
+
if vc_row is None:
|
| 215 |
+
raise HTTPException(status_code=404, detail="Session not found")
|
| 216 |
+
store.update_state(
|
| 217 |
+
session_id,
|
| 218 |
+
recent_acks=[],
|
| 219 |
+
recent_openers=[],
|
| 220 |
+
recent_closings=[],
|
| 221 |
+
)
|
| 222 |
+
return {"ok": True, "session_id": session_id}
|
| 223 |
+
|
| 224 |
+
return r
|
backend/app/persona_call/state.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Phase state machine.
|
| 3 |
+
|
| 4 |
+
Five phases, deterministic transitions:
|
| 5 |
+
|
| 6 |
+
opening β(reason expressed OR 'how are you' cycle done)ββΆ topic
|
| 7 |
+
topic β(closing trigger)ββββββββββββββββββββββββββββββΆ pre_closing
|
| 8 |
+
pre_closing β(user terminal)βββββββββββββββββββββββββββββββΆ closed
|
| 9 |
+
pre_closing β(user recants / new topic)βββββββββββββββββββΆ topic
|
| 10 |
+
closed (terminal)
|
| 11 |
+
|
| 12 |
+
Phase is the single source of truth for which paragraph
|
| 13 |
+
:mod:`directive` injects into the per-turn system suffix.
|
| 14 |
+
"""
|
| 15 |
+
from __future__ import annotations
|
| 16 |
+
|
| 17 |
+
from typing import Dict, Optional
|
| 18 |
+
|
| 19 |
+
from . import closing, context, store
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
OPENING = "opening"
|
| 23 |
+
TOPIC = "topic"
|
| 24 |
+
PRE_CLOSING = "pre_closing"
|
| 25 |
+
CLOSED = "closed"
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def advance_on_user_utterance(
|
| 29 |
+
*,
|
| 30 |
+
session_id: str,
|
| 31 |
+
user_text: str,
|
| 32 |
+
reason_fallback_turn: int,
|
| 33 |
+
) -> Dict[str, object]:
|
| 34 |
+
"""Called by the WS orchestrator BEFORE composing the directive.
|
| 35 |
+
|
| 36 |
+
Updates the phase, turn_index, caller_context, and reason flags
|
| 37 |
+
based on the just-received user utterance. Returns the refreshed
|
| 38 |
+
state row so the composer has the latest view.
|
| 39 |
+
"""
|
| 40 |
+
state = store.ensure_state(session_id)
|
| 41 |
+
turn_index = int(state.get("turn_index") or 0)
|
| 42 |
+
phase = state.get("phase") or OPENING
|
| 43 |
+
|
| 44 |
+
# Caller signal extraction β accumulate into caller_context.
|
| 45 |
+
prev_signal = context.CallerSignal.from_dict(
|
| 46 |
+
state.get("caller_context") if isinstance(
|
| 47 |
+
state.get("caller_context"), dict) else {}
|
| 48 |
+
)
|
| 49 |
+
fresh = context.classify_utterance(user_text)
|
| 50 |
+
merged = prev_signal.merge(fresh)
|
| 51 |
+
|
| 52 |
+
# Phase transition rules.
|
| 53 |
+
pre_closing_trigger = state.get("pre_closing_trigger")
|
| 54 |
+
|
| 55 |
+
if phase in (OPENING, TOPIC) and closing.is_pre_closing_trigger(user_text):
|
| 56 |
+
phase = PRE_CLOSING
|
| 57 |
+
pre_closing_trigger = user_text.strip()
|
| 58 |
+
elif phase == PRE_CLOSING and closing.is_terminal(user_text):
|
| 59 |
+
phase = CLOSED
|
| 60 |
+
elif phase == OPENING:
|
| 61 |
+
# Move to 'topic' if the user stated a reason, OR if we've
|
| 62 |
+
# already done one "how are you" cycle (2 user turns in
|
| 63 |
+
# opening without a reason).
|
| 64 |
+
if merged.reason_expressed:
|
| 65 |
+
phase = TOPIC
|
| 66 |
+
elif turn_index >= 2:
|
| 67 |
+
# After two turns in 'opening' without a stated reason,
|
| 68 |
+
# the phase-machine forces 'topic' β the persona will
|
| 69 |
+
# ask the reason-fallback question via the composer.
|
| 70 |
+
phase = TOPIC
|
| 71 |
+
|
| 72 |
+
# Persist.
|
| 73 |
+
updated = store.update_state(
|
| 74 |
+
session_id,
|
| 75 |
+
phase=phase,
|
| 76 |
+
turn_index=turn_index + 1,
|
| 77 |
+
caller_context=merged.to_dict(),
|
| 78 |
+
pre_closing_trigger=pre_closing_trigger,
|
| 79 |
+
)
|
| 80 |
+
return updated
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def mark_how_are_you_decision(session_id: str, *, skipped: bool) -> None:
|
| 84 |
+
"""Record that the composer made a 'how are you' decision for this
|
| 85 |
+
session β idempotent; the deciding composer path may run multiple
|
| 86 |
+
turns but should only fire once."""
|
| 87 |
+
store.update_state(session_id, skipped_how_are_you=skipped)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def mark_reason_fallback_asked(session_id: str) -> None:
|
| 91 |
+
"""Record that the composer already asked the reason-fallback
|
| 92 |
+
question, so it never asks twice."""
|
| 93 |
+
store.update_state(session_id, asked_reason_fallback=True)
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def get(session_id: str) -> Optional[Dict[str, object]]:
|
| 97 |
+
"""Thin pass-through to the store β avoids callers importing
|
| 98 |
+
store directly."""
|
| 99 |
+
return store.get_state(session_id)
|
backend/app/persona_call/store.py
ADDED
|
@@ -0,0 +1,241 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
SQLite persistence for persona_call.
|
| 3 |
+
|
| 4 |
+
Two tables, both additive and empty when the feature flag is off:
|
| 5 |
+
|
| 6 |
+
persona_call_state per-session runtime state (phase, turn_index,
|
| 7 |
+
ledgers, caller_context).
|
| 8 |
+
|
| 9 |
+
persona_call_directives shadow-mode audit: every composed directive
|
| 10 |
+
is appended here so P1 (compose but don't
|
| 11 |
+
apply) produces a reviewable trail.
|
| 12 |
+
|
| 13 |
+
Follows the same pattern as backend/app/voice_call/store.py and the
|
| 14 |
+
rest of HomePilot: direct sqlite3, per-call connections, lazy
|
| 15 |
+
``ensure_schema`` with a process-scoped guard so it's cheap after
|
| 16 |
+
first use.
|
| 17 |
+
"""
|
| 18 |
+
from __future__ import annotations
|
| 19 |
+
|
| 20 |
+
import json
|
| 21 |
+
import sqlite3
|
| 22 |
+
import time
|
| 23 |
+
from typing import Any, Dict, List, Optional
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _db_path() -> str:
|
| 27 |
+
from ..storage import _get_db_path
|
| 28 |
+
return _get_db_path()
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def _conn() -> sqlite3.Connection:
|
| 32 |
+
con = sqlite3.connect(_db_path())
|
| 33 |
+
con.row_factory = sqlite3.Row
|
| 34 |
+
return con
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
_SCHEMA_READY = False
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def ensure_schema() -> None:
|
| 41 |
+
"""Create persona_call tables if missing. Safe to call any number
|
| 42 |
+
of times. No-op on subsequent calls within the same process."""
|
| 43 |
+
global _SCHEMA_READY
|
| 44 |
+
if _SCHEMA_READY:
|
| 45 |
+
return
|
| 46 |
+
con = _conn()
|
| 47 |
+
cur = con.cursor()
|
| 48 |
+
cur.executescript(
|
| 49 |
+
"""
|
| 50 |
+
CREATE TABLE IF NOT EXISTS persona_call_state (
|
| 51 |
+
session_id TEXT PRIMARY KEY,
|
| 52 |
+
phase TEXT NOT NULL DEFAULT 'opening',
|
| 53 |
+
turn_index INTEGER NOT NULL DEFAULT 0,
|
| 54 |
+
skipped_how_are_you INTEGER NOT NULL DEFAULT 0,
|
| 55 |
+
asked_reason_fallback INTEGER NOT NULL DEFAULT 0,
|
| 56 |
+
reason_for_call TEXT NOT NULL DEFAULT '',
|
| 57 |
+
last_backchannel_ms INTEGER NOT NULL DEFAULT 0,
|
| 58 |
+
recent_acks TEXT NOT NULL DEFAULT '[]',
|
| 59 |
+
recent_openers TEXT NOT NULL DEFAULT '[]',
|
| 60 |
+
recent_closings TEXT NOT NULL DEFAULT '[]',
|
| 61 |
+
caller_context TEXT NOT NULL DEFAULT '{}',
|
| 62 |
+
pre_closing_trigger TEXT,
|
| 63 |
+
created_at INTEGER NOT NULL,
|
| 64 |
+
updated_at INTEGER NOT NULL
|
| 65 |
+
);
|
| 66 |
+
|
| 67 |
+
CREATE TABLE IF NOT EXISTS persona_call_directives (
|
| 68 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 69 |
+
session_id TEXT NOT NULL,
|
| 70 |
+
turn_index INTEGER NOT NULL,
|
| 71 |
+
phase TEXT NOT NULL,
|
| 72 |
+
applied INTEGER NOT NULL DEFAULT 1,
|
| 73 |
+
system_suffix TEXT NOT NULL,
|
| 74 |
+
post_directives TEXT NOT NULL DEFAULT '[]',
|
| 75 |
+
created_at INTEGER NOT NULL
|
| 76 |
+
);
|
| 77 |
+
CREATE INDEX IF NOT EXISTS ix_pcd_session
|
| 78 |
+
ON persona_call_directives (session_id, turn_index);
|
| 79 |
+
"""
|
| 80 |
+
)
|
| 81 |
+
con.commit()
|
| 82 |
+
con.close()
|
| 83 |
+
_SCHEMA_READY = True
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def _now_ms() -> int:
|
| 87 |
+
return int(time.time() * 1000)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
# ββ persona_call_state CRUD βββββββββββββββββββββββββββββββββββββββββββ
|
| 91 |
+
|
| 92 |
+
def get_state(session_id: str) -> Optional[Dict[str, Any]]:
|
| 93 |
+
ensure_schema()
|
| 94 |
+
con = _conn()
|
| 95 |
+
cur = con.cursor()
|
| 96 |
+
cur.execute(
|
| 97 |
+
"SELECT * FROM persona_call_state WHERE session_id = ?",
|
| 98 |
+
(session_id,),
|
| 99 |
+
)
|
| 100 |
+
row = cur.fetchone()
|
| 101 |
+
con.close()
|
| 102 |
+
if row is None:
|
| 103 |
+
return None
|
| 104 |
+
d = {k: row[k] for k in row.keys()}
|
| 105 |
+
# Deserialize JSON columns.
|
| 106 |
+
for col in ("recent_acks", "recent_openers", "recent_closings",
|
| 107 |
+
"caller_context"):
|
| 108 |
+
try:
|
| 109 |
+
d[col] = json.loads(d.get(col) or "[]")
|
| 110 |
+
except Exception:
|
| 111 |
+
d[col] = [] if col != "caller_context" else {}
|
| 112 |
+
return d
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def ensure_state(session_id: str) -> Dict[str, Any]:
|
| 116 |
+
"""Upsert-shaped: returns the existing row, or creates a fresh one
|
| 117 |
+
in the 'opening' phase if this is the first turn."""
|
| 118 |
+
existing = get_state(session_id)
|
| 119 |
+
if existing is not None:
|
| 120 |
+
return existing
|
| 121 |
+
now = _now_ms()
|
| 122 |
+
con = _conn()
|
| 123 |
+
cur = con.cursor()
|
| 124 |
+
cur.execute(
|
| 125 |
+
"""
|
| 126 |
+
INSERT OR IGNORE INTO persona_call_state
|
| 127 |
+
(session_id, created_at, updated_at)
|
| 128 |
+
VALUES (?, ?, ?)
|
| 129 |
+
""",
|
| 130 |
+
(session_id, now, now),
|
| 131 |
+
)
|
| 132 |
+
con.commit()
|
| 133 |
+
con.close()
|
| 134 |
+
return get_state(session_id) # type: ignore[return-value]
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def update_state(session_id: str, **changes: Any) -> Dict[str, Any]:
|
| 138 |
+
"""Apply a partial update. JSON-typed columns accept Python lists /
|
| 139 |
+
dicts; everything else is a scalar."""
|
| 140 |
+
if not changes:
|
| 141 |
+
return ensure_state(session_id)
|
| 142 |
+
|
| 143 |
+
json_cols = {"recent_acks", "recent_openers", "recent_closings",
|
| 144 |
+
"caller_context"}
|
| 145 |
+
sets, params = [], []
|
| 146 |
+
for col, val in changes.items():
|
| 147 |
+
if col in json_cols:
|
| 148 |
+
val = json.dumps(val, separators=(",", ":"))
|
| 149 |
+
if col in ("skipped_how_are_you", "asked_reason_fallback"):
|
| 150 |
+
val = 1 if val else 0
|
| 151 |
+
sets.append(f"{col} = ?")
|
| 152 |
+
params.append(val)
|
| 153 |
+
sets.append("updated_at = ?")
|
| 154 |
+
params.append(_now_ms())
|
| 155 |
+
params.append(session_id)
|
| 156 |
+
|
| 157 |
+
ensure_state(session_id)
|
| 158 |
+
con = _conn()
|
| 159 |
+
cur = con.cursor()
|
| 160 |
+
cur.execute(
|
| 161 |
+
f"UPDATE persona_call_state SET {', '.join(sets)} WHERE session_id = ?",
|
| 162 |
+
params,
|
| 163 |
+
)
|
| 164 |
+
con.commit()
|
| 165 |
+
con.close()
|
| 166 |
+
return get_state(session_id) # type: ignore[return-value]
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
# ββ persona_call_directives (shadow-mode audit) βββββββββββββββββββββββ
|
| 170 |
+
|
| 171 |
+
def record_directive(
|
| 172 |
+
*,
|
| 173 |
+
session_id: str,
|
| 174 |
+
turn_index: int,
|
| 175 |
+
phase: str,
|
| 176 |
+
applied: bool,
|
| 177 |
+
system_suffix: str,
|
| 178 |
+
post_directives: List[str],
|
| 179 |
+
) -> None:
|
| 180 |
+
ensure_schema()
|
| 181 |
+
con = _conn()
|
| 182 |
+
cur = con.cursor()
|
| 183 |
+
cur.execute(
|
| 184 |
+
"""
|
| 185 |
+
INSERT INTO persona_call_directives
|
| 186 |
+
(session_id, turn_index, phase, applied,
|
| 187 |
+
system_suffix, post_directives, created_at)
|
| 188 |
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
| 189 |
+
""",
|
| 190 |
+
(
|
| 191 |
+
session_id,
|
| 192 |
+
int(turn_index),
|
| 193 |
+
phase,
|
| 194 |
+
1 if applied else 0,
|
| 195 |
+
system_suffix,
|
| 196 |
+
json.dumps(post_directives, separators=(",", ":")),
|
| 197 |
+
_now_ms(),
|
| 198 |
+
),
|
| 199 |
+
)
|
| 200 |
+
con.commit()
|
| 201 |
+
con.close()
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def last_directive(session_id: str) -> Optional[Dict[str, Any]]:
|
| 205 |
+
ensure_schema()
|
| 206 |
+
con = _conn()
|
| 207 |
+
cur = con.cursor()
|
| 208 |
+
cur.execute(
|
| 209 |
+
"""
|
| 210 |
+
SELECT session_id, turn_index, phase, applied,
|
| 211 |
+
system_suffix, post_directives, created_at
|
| 212 |
+
FROM persona_call_directives
|
| 213 |
+
WHERE session_id = ?
|
| 214 |
+
ORDER BY id DESC
|
| 215 |
+
LIMIT 1
|
| 216 |
+
""",
|
| 217 |
+
(session_id,),
|
| 218 |
+
)
|
| 219 |
+
row = cur.fetchone()
|
| 220 |
+
con.close()
|
| 221 |
+
if row is None:
|
| 222 |
+
return None
|
| 223 |
+
d = {k: row[k] for k in row.keys()}
|
| 224 |
+
try:
|
| 225 |
+
d["post_directives"] = json.loads(d.get("post_directives") or "[]")
|
| 226 |
+
except Exception:
|
| 227 |
+
d["post_directives"] = []
|
| 228 |
+
d["applied"] = bool(d.get("applied"))
|
| 229 |
+
return d
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
# ββ test helper βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 233 |
+
|
| 234 |
+
def _reset_for_tests() -> None:
|
| 235 |
+
ensure_schema()
|
| 236 |
+
con = _conn()
|
| 237 |
+
cur = con.cursor()
|
| 238 |
+
cur.execute("DELETE FROM persona_call_directives")
|
| 239 |
+
cur.execute("DELETE FROM persona_call_state")
|
| 240 |
+
con.commit()
|
| 241 |
+
con.close()
|
backend/app/studio/render_jobs.py
CHANGED
|
@@ -53,6 +53,11 @@ class RenderJob:
|
|
| 53 |
duration_sec: float = 0.0
|
| 54 |
width: int = 0
|
| 55 |
height: int = 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
|
| 57 |
def to_dict(self) -> Dict[str, Any]:
|
| 58 |
d = asdict(self)
|
|
@@ -115,6 +120,9 @@ def submit_render(
|
|
| 115 |
kind: RenderKind,
|
| 116 |
preset: PlatformPreset,
|
| 117 |
scenes: List[SceneInput],
|
|
|
|
|
|
|
|
|
|
| 118 |
) -> RenderJob:
|
| 119 |
"""Queue a render and return its job record. Worker thread starts immediately."""
|
| 120 |
job_id = _new_job_id()
|
|
@@ -128,6 +136,9 @@ def submit_render(
|
|
| 128 |
preset=preset,
|
| 129 |
status="queued",
|
| 130 |
output_path=output_path,
|
|
|
|
|
|
|
|
|
|
| 131 |
)
|
| 132 |
_store(job)
|
| 133 |
|
|
@@ -169,6 +180,9 @@ def _run_job(job_id: str, scenes: List[SceneInput]) -> None:
|
|
| 169 |
kind=job.kind,
|
| 170 |
output_path=job.output_path or "",
|
| 171 |
on_progress=_on_progress,
|
|
|
|
|
|
|
|
|
|
| 172 |
)
|
| 173 |
_update(
|
| 174 |
job_id,
|
|
|
|
| 53 |
duration_sec: float = 0.0
|
| 54 |
width: int = 0
|
| 55 |
height: int = 0
|
| 56 |
+
# Render-time audio / subtitle options, captured so /jobs/{id}
|
| 57 |
+
# responses can describe exactly how the file was produced.
|
| 58 |
+
audio_rate: float = 1.0
|
| 59 |
+
audio_pitch: float = 1.0
|
| 60 |
+
subtitles: str = "none"
|
| 61 |
|
| 62 |
def to_dict(self) -> Dict[str, Any]:
|
| 63 |
d = asdict(self)
|
|
|
|
| 120 |
kind: RenderKind,
|
| 121 |
preset: PlatformPreset,
|
| 122 |
scenes: List[SceneInput],
|
| 123 |
+
audio_rate: float = 1.0,
|
| 124 |
+
audio_pitch: float = 1.0,
|
| 125 |
+
subtitles: str = "none",
|
| 126 |
) -> RenderJob:
|
| 127 |
"""Queue a render and return its job record. Worker thread starts immediately."""
|
| 128 |
job_id = _new_job_id()
|
|
|
|
| 136 |
preset=preset,
|
| 137 |
status="queued",
|
| 138 |
output_path=output_path,
|
| 139 |
+
audio_rate=audio_rate,
|
| 140 |
+
audio_pitch=audio_pitch,
|
| 141 |
+
subtitles=subtitles,
|
| 142 |
)
|
| 143 |
_store(job)
|
| 144 |
|
|
|
|
| 180 |
kind=job.kind,
|
| 181 |
output_path=job.output_path or "",
|
| 182 |
on_progress=_on_progress,
|
| 183 |
+
audio_rate=job.audio_rate,
|
| 184 |
+
audio_pitch=job.audio_pitch,
|
| 185 |
+
subtitles=job.subtitles,
|
| 186 |
)
|
| 187 |
_update(
|
| 188 |
job_id,
|
backend/app/studio/render_mp4.py
CHANGED
|
@@ -54,12 +54,17 @@ class SceneInput:
|
|
| 54 |
Either ``image_path`` or ``video_path`` should be provided. If both are
|
| 55 |
set, ``video_path`` wins (it is the higher-fidelity asset). If neither
|
| 56 |
is set, the scene is rendered as a black frame of ``duration_sec``.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
"""
|
| 58 |
idx: int
|
| 59 |
duration_sec: float = 5.0
|
| 60 |
image_path: Optional[str] = None
|
| 61 |
video_path: Optional[str] = None
|
| 62 |
audio_path: Optional[str] = None
|
|
|
|
| 63 |
|
| 64 |
|
| 65 |
@dataclass
|
|
@@ -243,6 +248,119 @@ def _per_scene_args(
|
|
| 243 |
return inputs, filters, f"v{scene.idx}"
|
| 244 |
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
def _has_audio(path: str) -> bool:
|
| 247 |
"""Probe a media file and return True iff it has at least one audio stream."""
|
| 248 |
try:
|
|
@@ -277,6 +395,9 @@ def render_scenes(
|
|
| 277 |
kind: RenderKind,
|
| 278 |
output_path: str,
|
| 279 |
on_progress: Optional[Callable[[float], None]] = None,
|
|
|
|
|
|
|
|
|
|
| 280 |
) -> RenderResult:
|
| 281 |
"""Render a list of scenes into a single MP4 at ``output_path``.
|
| 282 |
|
|
@@ -284,6 +405,18 @@ def render_scenes(
|
|
| 284 |
then concatenates the intermediates with ffmpeg's concat demuxer. This
|
| 285 |
is slower than a single filter graph but far more robust against scenes
|
| 286 |
with very different codecs / resolutions / sample rates / pixel formats.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
"""
|
| 288 |
if not ffmpeg_available():
|
| 289 |
raise RuntimeError(
|
|
@@ -294,6 +427,8 @@ def render_scenes(
|
|
| 294 |
|
| 295 |
width, height, fps = _resolution_for_preset(preset)
|
| 296 |
encode_args = _encode_args_for_kind(kind, preset)
|
|
|
|
|
|
|
| 297 |
|
| 298 |
out = Path(output_path)
|
| 299 |
out.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -325,24 +460,61 @@ def render_scenes(
|
|
| 325 |
image_path=image_local,
|
| 326 |
video_path=video_local,
|
| 327 |
audio_path=audio_local,
|
|
|
|
| 328 |
)
|
| 329 |
|
| 330 |
-
inputs, filters,
|
| 331 |
normalized, width, height, fps, scratch
|
| 332 |
)
|
| 333 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 334 |
# Audio handling: prefer explicit audio_path; otherwise reuse the
|
| 335 |
# video's audio track if present; otherwise generate silence so
|
| 336 |
# every clip has the same stream layout for the concat demuxer.
|
| 337 |
audio_inputs: list[str] = []
|
| 338 |
audio_filter: list[str] = []
|
| 339 |
audio_label = "a0"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 340 |
if audio_local:
|
| 341 |
audio_inputs += ["-i", audio_local]
|
| 342 |
# Index 1 is the audio file (after the single video input).
|
| 343 |
-
audio_filter.append(f"[1:a]
|
| 344 |
elif video_local and _has_audio(video_local):
|
| 345 |
-
audio_filter.append(f"[0:a]
|
| 346 |
else:
|
| 347 |
audio_inputs += [
|
| 348 |
"-f", "lavfi",
|
|
@@ -362,7 +534,7 @@ def render_scenes(
|
|
| 362 |
*inputs,
|
| 363 |
*audio_inputs,
|
| 364 |
"-filter_complex", filter_complex,
|
| 365 |
-
"-map", "[
|
| 366 |
"-map", f"[{audio_label}]",
|
| 367 |
"-r", str(fps),
|
| 368 |
"-shortest",
|
|
@@ -396,6 +568,36 @@ def render_scenes(
|
|
| 396 |
]
|
| 397 |
_run(concat_cmd)
|
| 398 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 399 |
if on_progress:
|
| 400 |
on_progress(100.0)
|
| 401 |
|
|
|
|
| 54 |
Either ``image_path`` or ``video_path`` should be provided. If both are
|
| 55 |
set, ``video_path`` wins (it is the higher-fidelity asset). If neither
|
| 56 |
is set, the scene is rendered as a black frame of ``duration_sec``.
|
| 57 |
+
|
| 58 |
+
``narration`` is the source text used to synthesize subtitles when
|
| 59 |
+
the caller passes ``subtitles="burn_in"`` to :func:`render_scenes`.
|
| 60 |
+
It does not affect rendering when subtitles are off.
|
| 61 |
"""
|
| 62 |
idx: int
|
| 63 |
duration_sec: float = 5.0
|
| 64 |
image_path: Optional[str] = None
|
| 65 |
video_path: Optional[str] = None
|
| 66 |
audio_path: Optional[str] = None
|
| 67 |
+
narration: str = ""
|
| 68 |
|
| 69 |
|
| 70 |
@dataclass
|
|
|
|
| 248 |
return inputs, filters, f"v{scene.idx}"
|
| 249 |
|
| 250 |
|
| 251 |
+
def _audio_transform_chain(rate: float, pitch: float) -> Optional[str]:
|
| 252 |
+
"""Build the ffmpeg audio-filter chain for the chosen rate + pitch.
|
| 253 |
+
|
| 254 |
+
Returns None when both are 1.0 (caller skips the transform stage).
|
| 255 |
+
|
| 256 |
+
Rate is applied with ``atempo``. Pitch uses the industry-standard
|
| 257 |
+
``asetrate=48000*pitch, atempo=1/pitch`` compound so the tempo is
|
| 258 |
+
preserved while the pitch shifts. When both are non-unit we stack the
|
| 259 |
+
pitch compound first, then an outer atempo for rate.
|
| 260 |
+
|
| 261 |
+
``atempo`` only accepts 0.5β2.0 per stage; we clamp inputs upstream
|
| 262 |
+
to stay within that range so we never need to chain multiple stages.
|
| 263 |
+
"""
|
| 264 |
+
rate = max(0.5, min(2.0, float(rate or 1.0)))
|
| 265 |
+
pitch = max(0.5, min(2.0, float(pitch or 1.0)))
|
| 266 |
+
if rate == 1.0 and pitch == 1.0:
|
| 267 |
+
return None
|
| 268 |
+
parts: list[str] = []
|
| 269 |
+
if pitch != 1.0:
|
| 270 |
+
parts.append(f"asetrate=48000*{pitch:.6f}")
|
| 271 |
+
parts.append(f"atempo={1.0 / pitch:.6f}")
|
| 272 |
+
if rate != 1.0:
|
| 273 |
+
parts.append(f"atempo={rate:.6f}")
|
| 274 |
+
return ",".join(parts)
|
| 275 |
+
|
| 276 |
+
|
| 277 |
+
def _ass_escape(text: str) -> str:
|
| 278 |
+
"""Escape a string for embedding inside a libass SRT/ASS cue."""
|
| 279 |
+
return (text or "").replace("\\", "\\\\").replace("{", "\\{").replace("}", "\\}")
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
def _srt_timestamp(seconds: float) -> str:
|
| 283 |
+
"""Convert seconds to an SRT timestamp (HH:MM:SS,mmm)."""
|
| 284 |
+
if seconds < 0:
|
| 285 |
+
seconds = 0.0
|
| 286 |
+
ms = int(round(seconds * 1000))
|
| 287 |
+
hours, ms = divmod(ms, 3_600_000)
|
| 288 |
+
minutes, ms = divmod(ms, 60_000)
|
| 289 |
+
secs, ms = divmod(ms, 1000)
|
| 290 |
+
return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}"
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def _split_narration_for_subtitles(text: str, max_chars: int = 42) -> list[str]:
|
| 294 |
+
"""Chunk narration into subtitle-sized lines.
|
| 295 |
+
|
| 296 |
+
Subtitles read best at ~42 chars/line (YouTube's auto-caption default).
|
| 297 |
+
We split on sentence boundaries first, then on commas when a chunk is
|
| 298 |
+
still too long, falling back to hard-wrap at word boundaries.
|
| 299 |
+
"""
|
| 300 |
+
import re
|
| 301 |
+
if not text:
|
| 302 |
+
return []
|
| 303 |
+
# Sentence split on ., !, ?, or newline.
|
| 304 |
+
sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+|\n+", text) if s.strip()]
|
| 305 |
+
chunks: list[str] = []
|
| 306 |
+
for s in sentences:
|
| 307 |
+
if len(s) <= max_chars:
|
| 308 |
+
chunks.append(s)
|
| 309 |
+
continue
|
| 310 |
+
# Try splitting on commas first.
|
| 311 |
+
sub = [p.strip() for p in s.split(",") if p.strip()]
|
| 312 |
+
if all(len(p) <= max_chars for p in sub):
|
| 313 |
+
chunks.extend(sub)
|
| 314 |
+
continue
|
| 315 |
+
# Fallback: greedy word-wrap.
|
| 316 |
+
cur = ""
|
| 317 |
+
for word in s.split():
|
| 318 |
+
if not cur:
|
| 319 |
+
cur = word
|
| 320 |
+
elif len(cur) + 1 + len(word) <= max_chars:
|
| 321 |
+
cur = f"{cur} {word}"
|
| 322 |
+
else:
|
| 323 |
+
chunks.append(cur)
|
| 324 |
+
cur = word
|
| 325 |
+
if cur:
|
| 326 |
+
chunks.append(cur)
|
| 327 |
+
return chunks
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
def _write_srt_for_scene(
|
| 331 |
+
narration: str,
|
| 332 |
+
duration_sec: float,
|
| 333 |
+
srt_path: Path,
|
| 334 |
+
*,
|
| 335 |
+
start_offset: float = 0.0,
|
| 336 |
+
) -> bool:
|
| 337 |
+
"""Write an SRT file whose cues span [0, duration_sec] for a single scene.
|
| 338 |
+
|
| 339 |
+
Returns True on success, False when the narration is empty (no SRT
|
| 340 |
+
should be emitted β and the caller should not try to burn an empty
|
| 341 |
+
subtitle file, which would fail).
|
| 342 |
+
"""
|
| 343 |
+
chunks = _split_narration_for_subtitles(narration)
|
| 344 |
+
if not chunks:
|
| 345 |
+
return False
|
| 346 |
+
dur = max(0.5, float(duration_sec or 5.0))
|
| 347 |
+
# Distribute the duration evenly across chunks. Cap per-cue at 5s so
|
| 348 |
+
# slow voices don't hold a single cue on screen forever.
|
| 349 |
+
per = min(dur / len(chunks), 5.0)
|
| 350 |
+
lines: list[str] = []
|
| 351 |
+
for i, chunk in enumerate(chunks):
|
| 352 |
+
start = start_offset + i * per
|
| 353 |
+
end = min(start_offset + dur, start + per)
|
| 354 |
+
if end <= start:
|
| 355 |
+
end = start + 0.1
|
| 356 |
+
lines.append(str(i + 1))
|
| 357 |
+
lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}")
|
| 358 |
+
lines.append(_ass_escape(chunk))
|
| 359 |
+
lines.append("")
|
| 360 |
+
srt_path.write_text("\n".join(lines), encoding="utf-8")
|
| 361 |
+
return True
|
| 362 |
+
|
| 363 |
+
|
| 364 |
def _has_audio(path: str) -> bool:
|
| 365 |
"""Probe a media file and return True iff it has at least one audio stream."""
|
| 366 |
try:
|
|
|
|
| 395 |
kind: RenderKind,
|
| 396 |
output_path: str,
|
| 397 |
on_progress: Optional[Callable[[float], None]] = None,
|
| 398 |
+
audio_rate: float = 1.0,
|
| 399 |
+
audio_pitch: float = 1.0,
|
| 400 |
+
subtitles: str = "none",
|
| 401 |
) -> RenderResult:
|
| 402 |
"""Render a list of scenes into a single MP4 at ``output_path``.
|
| 403 |
|
|
|
|
| 405 |
then concatenates the intermediates with ffmpeg's concat demuxer. This
|
| 406 |
is slower than a single filter graph but far more robust against scenes
|
| 407 |
with very different codecs / resolutions / sample rates / pixel formats.
|
| 408 |
+
|
| 409 |
+
Extra knobs:
|
| 410 |
+
|
| 411 |
+
- ``audio_rate`` (0.5β2.0, default 1.0): tempo multiplier for the
|
| 412 |
+
narration/audio track. Applied as ffmpeg ``atempo``.
|
| 413 |
+
- ``audio_pitch`` (0.5β2.0, default 1.0): pitch-shift multiplier.
|
| 414 |
+
Implemented via ``asetrate=48000*pitch, atempo=1/pitch`` so the
|
| 415 |
+
output duration is unchanged (tempo is preserved, pitch shifts).
|
| 416 |
+
- ``subtitles`` (``"burn_in"`` | ``"none"``, default ``"none"``):
|
| 417 |
+
when ``burn_in``, each scene's ``narration`` is chunked into an
|
| 418 |
+
SRT file and burned into the video via libass. A sidecar .srt
|
| 419 |
+
is also written next to the MP4 so editors can re-style.
|
| 420 |
"""
|
| 421 |
if not ffmpeg_available():
|
| 422 |
raise RuntimeError(
|
|
|
|
| 427 |
|
| 428 |
width, height, fps = _resolution_for_preset(preset)
|
| 429 |
encode_args = _encode_args_for_kind(kind, preset)
|
| 430 |
+
audio_transform = _audio_transform_chain(audio_rate, audio_pitch)
|
| 431 |
+
burn_subtitles = (subtitles or "none").lower() == "burn_in"
|
| 432 |
|
| 433 |
out = Path(output_path)
|
| 434 |
out.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 460 |
image_path=image_local,
|
| 461 |
video_path=video_local,
|
| 462 |
audio_path=audio_local,
|
| 463 |
+
narration=scene.narration,
|
| 464 |
)
|
| 465 |
|
| 466 |
+
inputs, filters, vlabel = _per_scene_args(
|
| 467 |
normalized, width, height, fps, scratch
|
| 468 |
)
|
| 469 |
|
| 470 |
+
# If subtitles are burning in, write a per-scene SRT into
|
| 471 |
+
# scratch and append a ``subtitles`` filter to the video chain.
|
| 472 |
+
# ffmpeg's subtitles filter uses libass under the hood. A
|
| 473 |
+
# sidecar .srt is also written next to the final MP4 below.
|
| 474 |
+
if burn_subtitles and scene.narration:
|
| 475 |
+
srt_path = scratch / f"scene_{i:04d}.srt"
|
| 476 |
+
wrote = _write_srt_for_scene(
|
| 477 |
+
scene.narration,
|
| 478 |
+
normalized.duration_sec,
|
| 479 |
+
srt_path,
|
| 480 |
+
)
|
| 481 |
+
if wrote:
|
| 482 |
+
# Escape the path for ffmpeg's filter-arg parser.
|
| 483 |
+
esc = str(srt_path).replace("\\", "/").replace(":", r"\:")
|
| 484 |
+
force_style = (
|
| 485 |
+
"Fontname=DejaVu Sans,Fontsize=28,"
|
| 486 |
+
"PrimaryColour=&H00FFFFFF&,"
|
| 487 |
+
"OutlineColour=&H80000000&,"
|
| 488 |
+
"BorderStyle=3,Outline=2,Shadow=0,MarginV=48"
|
| 489 |
+
)
|
| 490 |
+
# Replace the last video-chain step's output label so
|
| 491 |
+
# the subtitle filter sits at the end of the pipeline.
|
| 492 |
+
new_last = (
|
| 493 |
+
f"[{vlabel}]subtitles='{esc}':"
|
| 494 |
+
f"force_style='{force_style}'[{vlabel}_sub]"
|
| 495 |
+
)
|
| 496 |
+
filters.append(new_last)
|
| 497 |
+
vlabel = f"{vlabel}_sub"
|
| 498 |
+
|
| 499 |
# Audio handling: prefer explicit audio_path; otherwise reuse the
|
| 500 |
# video's audio track if present; otherwise generate silence so
|
| 501 |
# every clip has the same stream layout for the concat demuxer.
|
| 502 |
audio_inputs: list[str] = []
|
| 503 |
audio_filter: list[str] = []
|
| 504 |
audio_label = "a0"
|
| 505 |
+
# Common tail applied to every audio path: resample to 48 kHz
|
| 506 |
+
# stereo + apply the optional rate/pitch transform the caller
|
| 507 |
+
# requested. Consolidating the tail here keeps the three
|
| 508 |
+
# branches below in sync.
|
| 509 |
+
tail = "aresample=48000,aformat=channel_layouts=stereo"
|
| 510 |
+
if audio_transform:
|
| 511 |
+
tail = f"{tail},{audio_transform}"
|
| 512 |
if audio_local:
|
| 513 |
audio_inputs += ["-i", audio_local]
|
| 514 |
# Index 1 is the audio file (after the single video input).
|
| 515 |
+
audio_filter.append(f"[1:a]{tail}[{audio_label}]")
|
| 516 |
elif video_local and _has_audio(video_local):
|
| 517 |
+
audio_filter.append(f"[0:a]{tail}[{audio_label}]")
|
| 518 |
else:
|
| 519 |
audio_inputs += [
|
| 520 |
"-f", "lavfi",
|
|
|
|
| 534 |
*inputs,
|
| 535 |
*audio_inputs,
|
| 536 |
"-filter_complex", filter_complex,
|
| 537 |
+
"-map", f"[{vlabel}]",
|
| 538 |
"-map", f"[{audio_label}]",
|
| 539 |
"-r", str(fps),
|
| 540 |
"-shortest",
|
|
|
|
| 568 |
]
|
| 569 |
_run(concat_cmd)
|
| 570 |
|
| 571 |
+
# Sidecar SRT: full-video timeline, cue timings chained across
|
| 572 |
+
# scenes. Written whenever burn_subtitles=True, so editors get the
|
| 573 |
+
# same timing they saw burned in. Named <output>.srt.
|
| 574 |
+
if burn_subtitles:
|
| 575 |
+
sidecar = out.with_suffix(".srt")
|
| 576 |
+
running_offset = 0.0
|
| 577 |
+
cue_idx = 1
|
| 578 |
+
lines: list[str] = []
|
| 579 |
+
for s in scenes:
|
| 580 |
+
dur = max(0.5, float(s.duration_sec or 5.0))
|
| 581 |
+
chunks = _split_narration_for_subtitles(s.narration or "")
|
| 582 |
+
if chunks:
|
| 583 |
+
per = min(dur / len(chunks), 5.0)
|
| 584 |
+
for j, chunk in enumerate(chunks):
|
| 585 |
+
start = running_offset + j * per
|
| 586 |
+
end = min(running_offset + dur, start + per)
|
| 587 |
+
if end <= start:
|
| 588 |
+
end = start + 0.1
|
| 589 |
+
lines.append(str(cue_idx))
|
| 590 |
+
lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}")
|
| 591 |
+
lines.append(_ass_escape(chunk))
|
| 592 |
+
lines.append("")
|
| 593 |
+
cue_idx += 1
|
| 594 |
+
running_offset += dur
|
| 595 |
+
try:
|
| 596 |
+
sidecar.write_text("\n".join(lines), encoding="utf-8")
|
| 597 |
+
except OSError:
|
| 598 |
+
# Non-fatal β the burned-in subs are still present.
|
| 599 |
+
pass
|
| 600 |
+
|
| 601 |
if on_progress:
|
| 602 |
on_progress(100.0)
|
| 603 |
|
backend/app/studio/routes.py
CHANGED
|
@@ -2360,8 +2360,24 @@ class VideoMp4ExportRequest(BaseModel):
|
|
| 2360 |
``kind`` selects the encode profile; the platform preset (canvas
|
| 2361 |
aspect, fps) is read from the StudioVideo record so the same project
|
| 2362 |
always renders to the same canvas regardless of the chosen profile.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2363 |
"""
|
| 2364 |
kind: Literal["mp4_plain", "mp4_youtube"] = "mp4_plain"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2365 |
|
| 2366 |
|
| 2367 |
@router.post("/videos/{video_id}/export/mp4", status_code=202)
|
|
@@ -2394,6 +2410,8 @@ def video_export_mp4(
|
|
| 2394 |
raise HTTPException(status_code=400, detail="Video has no scenes to render")
|
| 2395 |
|
| 2396 |
# Translate StudioScene records into the renderer's SceneInput shape.
|
|
|
|
|
|
|
| 2397 |
inputs = [
|
| 2398 |
SceneInput(
|
| 2399 |
idx=i,
|
|
@@ -2401,20 +2419,100 @@ def video_export_mp4(
|
|
| 2401 |
image_path=s.imageUrl,
|
| 2402 |
video_path=s.videoUrl,
|
| 2403 |
audio_path=s.audioUrl,
|
|
|
|
| 2404 |
)
|
| 2405 |
for i, s in enumerate(scenes)
|
| 2406 |
]
|
| 2407 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2408 |
job = render_jobs.submit_render(
|
| 2409 |
user_id=user["id"],
|
| 2410 |
video_id=video_id,
|
| 2411 |
kind=req.kind,
|
| 2412 |
preset=video.platformPreset,
|
| 2413 |
scenes=inputs,
|
|
|
|
|
|
|
|
|
|
| 2414 |
)
|
| 2415 |
return job.to_dict()
|
| 2416 |
|
| 2417 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2418 |
@router.get("/videos/{video_id}/export/jobs/{job_id}")
|
| 2419 |
def video_export_job_status(
|
| 2420 |
video_id: str,
|
|
|
|
| 2360 |
``kind`` selects the encode profile; the platform preset (canvas
|
| 2361 |
aspect, fps) is read from the StudioVideo record so the same project
|
| 2362 |
always renders to the same canvas regardless of the chosen profile.
|
| 2363 |
+
|
| 2364 |
+
Audio / subtitle options are additive: the defaults reproduce the
|
| 2365 |
+
original concat-only behavior so the endpoint stays backward
|
| 2366 |
+
compatible for clients that don't send them.
|
| 2367 |
"""
|
| 2368 |
kind: Literal["mp4_plain", "mp4_youtube"] = "mp4_plain"
|
| 2369 |
+
# Audio rate (tempo) multiplier applied during render, 0.5β2.0. 1.0 is
|
| 2370 |
+
# untouched; <1 slows narration, >1 speeds it up. Duration of the
|
| 2371 |
+
# output clip stretches / shrinks accordingly.
|
| 2372 |
+
audio_rate: float = 1.0
|
| 2373 |
+
# Audio pitch shift, 0.5β2.0. 1.0 is untouched. Implemented with the
|
| 2374 |
+
# industry-standard asetrate + atempo compound so tempo is preserved
|
| 2375 |
+
# while only pitch shifts.
|
| 2376 |
+
audio_pitch: float = 1.0
|
| 2377 |
+
# When "burn_in", scene narration is wrapped into timed SRT cues and
|
| 2378 |
+
# burned into the video via ffmpeg's libass subtitle filter. A sidecar
|
| 2379 |
+
# .srt is always written alongside the MP4 when subtitles != "none".
|
| 2380 |
+
subtitles: Literal["burn_in", "none"] = "none"
|
| 2381 |
|
| 2382 |
|
| 2383 |
@router.post("/videos/{video_id}/export/mp4", status_code=202)
|
|
|
|
| 2410 |
raise HTTPException(status_code=400, detail="Video has no scenes to render")
|
| 2411 |
|
| 2412 |
# Translate StudioScene records into the renderer's SceneInput shape.
|
| 2413 |
+
# ``narration`` is the source text for subtitles; it is the same field
|
| 2414 |
+
# the Piper WASM wizard used to synthesize the uploaded audioUrl.
|
| 2415 |
inputs = [
|
| 2416 |
SceneInput(
|
| 2417 |
idx=i,
|
|
|
|
| 2419 |
image_path=s.imageUrl,
|
| 2420 |
video_path=s.videoUrl,
|
| 2421 |
audio_path=s.audioUrl,
|
| 2422 |
+
narration=s.narration or "",
|
| 2423 |
)
|
| 2424 |
for i, s in enumerate(scenes)
|
| 2425 |
]
|
| 2426 |
|
| 2427 |
+
# Clamp audio-transform inputs. The renderer itself has additional
|
| 2428 |
+
# defensive clamps, but rejecting absurd values at the API boundary
|
| 2429 |
+
# makes error messages clearer.
|
| 2430 |
+
audio_rate = max(0.5, min(2.0, float(req.audio_rate or 1.0)))
|
| 2431 |
+
audio_pitch = max(0.5, min(2.0, float(req.audio_pitch or 1.0)))
|
| 2432 |
+
|
| 2433 |
job = render_jobs.submit_render(
|
| 2434 |
user_id=user["id"],
|
| 2435 |
video_id=video_id,
|
| 2436 |
kind=req.kind,
|
| 2437 |
preset=video.platformPreset,
|
| 2438 |
scenes=inputs,
|
| 2439 |
+
audio_rate=audio_rate,
|
| 2440 |
+
audio_pitch=audio_pitch,
|
| 2441 |
+
subtitles=req.subtitles,
|
| 2442 |
)
|
| 2443 |
return job.to_dict()
|
| 2444 |
|
| 2445 |
|
| 2446 |
+
# ----------------------------------------------------------------------------
|
| 2447 |
+
# Narration upload
|
| 2448 |
+
# ----------------------------------------------------------------------------
|
| 2449 |
+
# The Creator Studio export wizard synthesizes per-scene narration with the
|
| 2450 |
+
# Piper WASM engine (in-browser), then uploads each WAV here. The server
|
| 2451 |
+
# persists the file under UPLOAD_DIR/studio/narration/<scene_id>.wav and
|
| 2452 |
+
# writes scene.audioUrl. During render the concat pipeline mixes this audio
|
| 2453 |
+
# into the final MP4.
|
| 2454 |
+
|
| 2455 |
+
from fastapi import UploadFile, File
|
| 2456 |
+
|
| 2457 |
+
|
| 2458 |
+
@router.post("/videos/{video_id}/scenes/{scene_id}/narration")
|
| 2459 |
+
async def upload_scene_narration(
|
| 2460 |
+
video_id: str,
|
| 2461 |
+
scene_id: str,
|
| 2462 |
+
file: UploadFile = File(..., description="Audio file (Piper-synthesized WAV)"),
|
| 2463 |
+
authorization: str = Header(default=""),
|
| 2464 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 2465 |
+
):
|
| 2466 |
+
"""Accept a narration audio upload for a scene. Writes ``scene.audioUrl``."""
|
| 2467 |
+
_user = _resolve_user_or_401(authorization, homepilot_session)
|
| 2468 |
+
|
| 2469 |
+
video = get_video(video_id)
|
| 2470 |
+
if not video:
|
| 2471 |
+
raise HTTPException(status_code=404, detail="Video not found")
|
| 2472 |
+
scene = get_scene(scene_id)
|
| 2473 |
+
if not scene or scene.videoId != video_id:
|
| 2474 |
+
raise HTTPException(status_code=404, detail="Scene not found")
|
| 2475 |
+
|
| 2476 |
+
# Piper produces 16-bit PCM WAV. Accept common audio types; the
|
| 2477 |
+
# renderer normalizes anyway.
|
| 2478 |
+
allowed_prefixes = ("audio/", "application/octet-stream")
|
| 2479 |
+
ctype = (file.content_type or "").lower()
|
| 2480 |
+
if ctype and not any(ctype.startswith(p) for p in allowed_prefixes):
|
| 2481 |
+
raise HTTPException(
|
| 2482 |
+
status_code=415,
|
| 2483 |
+
detail=f"Unsupported narration content-type: {file.content_type}",
|
| 2484 |
+
)
|
| 2485 |
+
|
| 2486 |
+
# Cap the upload at 50 MB β a minute of 48 kHz 16-bit stereo WAV is
|
| 2487 |
+
# ~11 MB, so this is generous even for long scenes.
|
| 2488 |
+
MAX_BYTES = 50 * 1024 * 1024
|
| 2489 |
+
payload = await file.read(MAX_BYTES + 1)
|
| 2490 |
+
if len(payload) > MAX_BYTES:
|
| 2491 |
+
raise HTTPException(status_code=413, detail="Narration file too large (>50 MB)")
|
| 2492 |
+
if not payload:
|
| 2493 |
+
raise HTTPException(status_code=400, detail="Empty narration file")
|
| 2494 |
+
|
| 2495 |
+
from ..config import UPLOAD_DIR
|
| 2496 |
+
narration_dir = os.path.join(UPLOAD_DIR, "studio", "narration")
|
| 2497 |
+
os.makedirs(narration_dir, exist_ok=True)
|
| 2498 |
+
# Use the scene_id as the filename so re-uploads replace the previous
|
| 2499 |
+
# take cleanly and there is no growing pile of orphan wav files.
|
| 2500 |
+
dest_path = os.path.join(narration_dir, f"{scene_id}.wav")
|
| 2501 |
+
with open(dest_path, "wb") as fh:
|
| 2502 |
+
fh.write(payload)
|
| 2503 |
+
|
| 2504 |
+
# Write back via the existing scene update path so everything
|
| 2505 |
+
# downstream (frontend list, exporter) reads a consistent record.
|
| 2506 |
+
update_scene(scene_id, StudioSceneUpdate(audioUrl=dest_path))
|
| 2507 |
+
|
| 2508 |
+
return {
|
| 2509 |
+
"ok": True,
|
| 2510 |
+
"scene_id": scene_id,
|
| 2511 |
+
"audioUrl": dest_path,
|
| 2512 |
+
"bytes": len(payload),
|
| 2513 |
+
}
|
| 2514 |
+
|
| 2515 |
+
|
| 2516 |
@router.get("/videos/{video_id}/export/jobs/{job_id}")
|
| 2517 |
def video_export_job_status(
|
| 2518 |
video_id: str,
|
backend/app/voice_call/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
HomePilot voice-call β additive backend module.
|
| 3 |
+
|
| 4 |
+
Scope (MVP M1)
|
| 5 |
+
--------------
|
| 6 |
+
Purely additive feature. With ``VOICE_CALL_ENABLED=false`` (the default)
|
| 7 |
+
this package is never imported into the running app and the DB tables it
|
| 8 |
+
needs are never created β dropping the module directory is sufficient to
|
| 9 |
+
remove the feature entirely.
|
| 10 |
+
|
| 11 |
+
Public entry point is :func:`voice_call.router.build_router`; everything
|
| 12 |
+
else is internal. No existing backend module imports from this package.
|
| 13 |
+
"""
|
| 14 |
+
from __future__ import annotations
|
| 15 |
+
|
| 16 |
+
__all__ = ["config", "router"]
|
backend/app/voice_call/barge_in.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Per-session barge-in token registry.
|
| 3 |
+
|
| 4 |
+
One process-local ``dict`` keyed by ``session_id``; each entry carries
|
| 5 |
+
a single :class:`BargeInToken` representing the currently-active
|
| 6 |
+
assistant turn. The token exposes an ``asyncio.Event`` the streaming
|
| 7 |
+
turn runner polls between chunks β when the client sends
|
| 8 |
+
``user.barge_in`` the WS handler calls :func:`cancel_active` which
|
| 9 |
+
flips the event and the generator exits cleanly on its next poll.
|
| 10 |
+
|
| 11 |
+
The module is deliberately small and state-machine-only:
|
| 12 |
+
|
| 13 |
+
* no HTTP, no DB, no asyncio transport β unit-testable in
|
| 14 |
+
milliseconds,
|
| 15 |
+
* ``cancel_active`` refuses cancels for stale ``turn_id`` values so
|
| 16 |
+
a barge-in racing a new turn is a silent no-op,
|
| 17 |
+
* ``cancel`` is idempotent (``asyncio.Event.set()`` is idempotent),
|
| 18 |
+
* ``clear`` wipes a session on WS close so a resume starts fresh.
|
| 19 |
+
|
| 20 |
+
Contract is documented in
|
| 21 |
+
``docs/analysis/voice-call-streaming-design.md`` Β§ 4.3.
|
| 22 |
+
"""
|
| 23 |
+
from __future__ import annotations
|
| 24 |
+
|
| 25 |
+
import asyncio
|
| 26 |
+
import threading
|
| 27 |
+
from dataclasses import dataclass, field
|
| 28 |
+
from typing import Dict, Optional
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
@dataclass
|
| 32 |
+
class BargeInToken:
|
| 33 |
+
"""Per-turn cancellation handle.
|
| 34 |
+
|
| 35 |
+
Created by :func:`new_token` when the WS handler starts streaming
|
| 36 |
+
an assistant turn; polled by the streaming runner; flipped by
|
| 37 |
+
:func:`cancel_active` when the user barges in.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
turn_id: str
|
| 41 |
+
_ev: asyncio.Event = field(default_factory=asyncio.Event)
|
| 42 |
+
|
| 43 |
+
def cancel(self) -> None:
|
| 44 |
+
"""Flip the event. Idempotent."""
|
| 45 |
+
self._ev.set()
|
| 46 |
+
|
| 47 |
+
def is_cancelled(self) -> bool:
|
| 48 |
+
"""Cheap read the streaming runner calls between chunks."""
|
| 49 |
+
return self._ev.is_set()
|
| 50 |
+
|
| 51 |
+
async def wait(self) -> None:
|
| 52 |
+
"""Optional helper for callers that want to ``asyncio.wait``
|
| 53 |
+
on cancel rather than poll."""
|
| 54 |
+
await self._ev.wait()
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
# ββ process-local registry ββββββββββββββββββββββββββββββββββββββββββββ
|
| 58 |
+
# One active token per session. A single lock because the hot path
|
| 59 |
+
# (``cancel_active``) is O(1) β no contention worth sharding for.
|
| 60 |
+
|
| 61 |
+
_lock = threading.Lock()
|
| 62 |
+
_active: Dict[str, BargeInToken] = {}
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def new_token(session_id: str, turn_id: str) -> BargeInToken:
|
| 66 |
+
"""Register ``turn_id`` as the active turn for ``session_id``.
|
| 67 |
+
|
| 68 |
+
Any previous active token for this session is replaced β if a
|
| 69 |
+
turn started without the previous one clearing (which would be
|
| 70 |
+
a bug upstream) we fail gracefully rather than leak.
|
| 71 |
+
"""
|
| 72 |
+
token = BargeInToken(turn_id=turn_id)
|
| 73 |
+
with _lock:
|
| 74 |
+
_active[session_id] = token
|
| 75 |
+
return token
|
| 76 |
+
|
| 77 |
+
|
| 78 |
+
def cancel_active(session_id: str, turn_id: str) -> bool:
|
| 79 |
+
"""Cancel the active turn for ``session_id`` IFF its ``turn_id``
|
| 80 |
+
matches the argument.
|
| 81 |
+
|
| 82 |
+
Returns True on a successful cancel, False if the active turn has
|
| 83 |
+
a different id (stale barge-in) or no turn is active.
|
| 84 |
+
"""
|
| 85 |
+
with _lock:
|
| 86 |
+
token = _active.get(session_id)
|
| 87 |
+
if token is None or token.turn_id != turn_id:
|
| 88 |
+
return False
|
| 89 |
+
# Release the lock before setting β the event set is itself
|
| 90 |
+
# thread-safe, and keeping the lock narrow keeps the hot path
|
| 91 |
+
# fast under contention.
|
| 92 |
+
token.cancel()
|
| 93 |
+
return True
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
def get_active(session_id: str) -> Optional[BargeInToken]:
|
| 97 |
+
"""Read the currently-active token without mutating.
|
| 98 |
+
|
| 99 |
+
Used by the WS handler to find the active ``turn_id`` when the
|
| 100 |
+
client sends a ``transcript.partial`` (secondary barge-in signal)
|
| 101 |
+
without an explicit ``turn_id``.
|
| 102 |
+
"""
|
| 103 |
+
with _lock:
|
| 104 |
+
return _active.get(session_id)
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def clear_session(session_id: str) -> None:
|
| 108 |
+
"""Drop all tokens for ``session_id`` β called on WS close.
|
| 109 |
+
|
| 110 |
+
If a turn was mid-stream we cancel it first so the streaming
|
| 111 |
+
runner exits promptly; the handler is responsible for emitting
|
| 112 |
+
the final envelope.
|
| 113 |
+
"""
|
| 114 |
+
with _lock:
|
| 115 |
+
token = _active.pop(session_id, None)
|
| 116 |
+
if token is not None and not token.is_cancelled():
|
| 117 |
+
token.cancel()
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _reset_for_tests() -> None:
|
| 121 |
+
"""Test hook β wipe the whole registry between cases."""
|
| 122 |
+
with _lock:
|
| 123 |
+
_active.clear()
|
backend/app/voice_call/config.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
voice_call runtime configuration.
|
| 3 |
+
|
| 4 |
+
Every knob is an environment variable with a safe default. The feature
|
| 5 |
+
ships **off by default** β set ``VOICE_CALL_ENABLED=true`` to opt in.
|
| 6 |
+
|
| 7 |
+
Privacy defaults are deliberately conservative per the review note
|
| 8 |
+
("treat storage/privacy as first-class from day 1"):
|
| 9 |
+
- ``VOICE_CALL_STORE_TRANSCRIPTS`` defaults to ``false``.
|
| 10 |
+
- Metadata persistence (sessions + events) is enabled by default, but
|
| 11 |
+
events only record event *types*, not full audio or raw user text.
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
from dataclasses import dataclass
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def _bool_env(name: str, default: bool) -> bool:
|
| 20 |
+
raw = os.getenv(name)
|
| 21 |
+
if raw is None:
|
| 22 |
+
return default
|
| 23 |
+
return raw.strip().lower() in ("1", "true", "yes", "on")
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def _int_env(name: str, default: int) -> int:
|
| 27 |
+
raw = os.getenv(name)
|
| 28 |
+
if raw is None:
|
| 29 |
+
return default
|
| 30 |
+
try:
|
| 31 |
+
return int(raw)
|
| 32 |
+
except (TypeError, ValueError):
|
| 33 |
+
return default
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
@dataclass(frozen=True)
|
| 37 |
+
class VoiceCallConfig:
|
| 38 |
+
"""Resolved runtime knobs. Immutable; re-import to refresh in tests."""
|
| 39 |
+
|
| 40 |
+
# Master flag. When false, ``router.build_router()`` returns an empty
|
| 41 |
+
# router and ``store.migrate()`` is a no-op β the feature is entirely
|
| 42 |
+
# latent.
|
| 43 |
+
enabled: bool
|
| 44 |
+
|
| 45 |
+
# WebSocket transport for the turn bridge. Off => HTTP-only polling
|
| 46 |
+
# fallback (not yet implemented, so if disabled the feature is
|
| 47 |
+
# effectively off even if ``enabled`` is true).
|
| 48 |
+
websocket_enabled: bool
|
| 49 |
+
|
| 50 |
+
# Provider strategy. Only "internal" is wired at MVP; the string is
|
| 51 |
+
# reserved for future external-realtime adapters.
|
| 52 |
+
provider: str
|
| 53 |
+
|
| 54 |
+
# Policy caps (server-enforced β never trust the client).
|
| 55 |
+
max_duration_sec: int
|
| 56 |
+
idle_timeout_sec: int
|
| 57 |
+
resume_window_sec: int
|
| 58 |
+
max_concurrent_per_user: int
|
| 59 |
+
session_create_rate_per_min: int
|
| 60 |
+
|
| 61 |
+
# Privacy. Turn transcripts are NOT persisted by default.
|
| 62 |
+
store_transcripts: bool
|
| 63 |
+
artifact_retention_days: int
|
| 64 |
+
|
| 65 |
+
# Phase 2 + 3 streaming (token-level LLM streaming + barge-in).
|
| 66 |
+
# Defaults off β when false, every byte on the wire and every
|
| 67 |
+
# code path taken is identical to today. Independent of
|
| 68 |
+
# ``enabled`` / ``websocket_enabled`` so the streaming path
|
| 69 |
+
# can be dark-shipped alongside unary turns for measurement.
|
| 70 |
+
streaming_enabled: bool
|
| 71 |
+
# Barge-in is an optional sub-feature of streaming; can be
|
| 72 |
+
# disabled independently if the VAD tap is misbehaving on a
|
| 73 |
+
# given deployment.
|
| 74 |
+
barge_in_enabled: bool
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def load() -> VoiceCallConfig:
|
| 78 |
+
"""Resolve config from env vars every call. Cheap; keep fresh for tests."""
|
| 79 |
+
return VoiceCallConfig(
|
| 80 |
+
enabled=_bool_env("VOICE_CALL_ENABLED", False),
|
| 81 |
+
websocket_enabled=_bool_env("VOICE_CALL_WEBSOCKET_ENABLED", True),
|
| 82 |
+
provider=os.getenv("VOICE_CALL_PROVIDER", "internal").strip() or "internal",
|
| 83 |
+
max_duration_sec=_int_env("VOICE_CALL_MAX_DURATION_SEC", 45 * 60),
|
| 84 |
+
idle_timeout_sec=_int_env("VOICE_CALL_IDLE_TIMEOUT_SEC", 30),
|
| 85 |
+
resume_window_sec=_int_env("VOICE_CALL_RESUME_WINDOW_SEC", 20),
|
| 86 |
+
max_concurrent_per_user=_int_env("VOICE_CALL_MAX_CONCURRENT", 2),
|
| 87 |
+
session_create_rate_per_min=_int_env("VOICE_CALL_CREATE_RATE", 10),
|
| 88 |
+
store_transcripts=_bool_env("VOICE_CALL_STORE_TRANSCRIPTS", False),
|
| 89 |
+
artifact_retention_days=_int_env("VOICE_CALL_ARTIFACT_TTL_DAYS", 30),
|
| 90 |
+
streaming_enabled=_bool_env("VOICE_CALL_STREAMING_ENABLED", False),
|
| 91 |
+
barge_in_enabled=_bool_env("VOICE_CALL_BARGE_IN_ENABLED", True),
|
| 92 |
+
)
|
backend/app/voice_call/models.py
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
voice_call request / response / event schemas.
|
| 3 |
+
|
| 4 |
+
Kept small on purpose: the public surface is half a dozen shapes. If a
|
| 5 |
+
new field is needed, add it optional so the client stays forward-
|
| 6 |
+
compatible with older backends (same rule the rest of HomePilot follows).
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from typing import Any, Dict, List, Literal, Optional
|
| 11 |
+
from pydantic import BaseModel, Field
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
EntryMode = Literal["header_button", "long_press", "hint_tap", "resume"]
|
| 15 |
+
|
| 16 |
+
SessionStatus = Literal[
|
| 17 |
+
"connecting",
|
| 18 |
+
"live",
|
| 19 |
+
"interrupted", # WS dropped, within resume window
|
| 20 |
+
"ending",
|
| 21 |
+
"ended",
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
class DeviceInfo(BaseModel):
|
| 26 |
+
platform: Optional[str] = None # "web" / "ios" / "android" / "desktop"
|
| 27 |
+
app_version: Optional[str] = None
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
class CreateSessionReq(BaseModel):
|
| 31 |
+
conversation_id: Optional[str] = None
|
| 32 |
+
persona_id: Optional[str] = None
|
| 33 |
+
entry_mode: EntryMode = "header_button"
|
| 34 |
+
device_info: DeviceInfo = Field(default_factory=DeviceInfo)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
class Capabilities(BaseModel):
|
| 38 |
+
"""What the server promises this session can do. Keep truthful β any
|
| 39 |
+
false promise here turns into a UX bug users feel immediately."""
|
| 40 |
+
# At MVP turn-based only. Full duplex (true barge-in) reads as false
|
| 41 |
+
# until we add server-side audio, per the review note.
|
| 42 |
+
interruptions: bool = False
|
| 43 |
+
barge_in: bool = False
|
| 44 |
+
# Assistant replies come as ``transcript.final`` only β no partials.
|
| 45 |
+
# This matches the current ``/v1/chat/completions`` behavior which
|
| 46 |
+
# rejects streaming with a 501 (openai_compat_endpoint.py:477).
|
| 47 |
+
transcript_live: bool = False
|
| 48 |
+
# Phase 2/3 opt-in β set True on the response when
|
| 49 |
+
# ``VOICE_CALL_STREAMING_ENABLED`` is on. Unaware clients ignore
|
| 50 |
+
# the field and stay on the unary path.
|
| 51 |
+
streaming: bool = False
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
class CreateSessionResp(BaseModel):
|
| 55 |
+
session_id: str
|
| 56 |
+
ws_url: str
|
| 57 |
+
resume_token: str # short-lived; only accepted by ws handshake
|
| 58 |
+
expires_at: int # epoch ms; WS must open before this
|
| 59 |
+
max_duration_sec: int
|
| 60 |
+
capabilities: Capabilities
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
class Session(BaseModel):
|
| 64 |
+
"""Public session record. Mirrors the DB row without resume_token or
|
| 65 |
+
raw payloads, so an accidental log of this shape never leaks
|
| 66 |
+
credentials."""
|
| 67 |
+
id: str
|
| 68 |
+
user_id: str
|
| 69 |
+
conversation_id: Optional[str] = None
|
| 70 |
+
persona_id: Optional[str] = None
|
| 71 |
+
entry_mode: EntryMode
|
| 72 |
+
status: SessionStatus
|
| 73 |
+
started_at: int
|
| 74 |
+
ended_at: Optional[int] = None
|
| 75 |
+
ended_reason: Optional[str] = None
|
| 76 |
+
duration_sec: Optional[int] = None
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
class EndSessionResp(BaseModel):
|
| 80 |
+
session_id: str
|
| 81 |
+
status: SessionStatus
|
| 82 |
+
ended_at: int
|
| 83 |
+
ended_reason: str
|
| 84 |
+
duration_sec: int
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
class HintsResp(BaseModel):
|
| 88 |
+
"""Discoverability copy + timings. Server is the source of truth so
|
| 89 |
+
the product team can tune thresholds without a frontend ship."""
|
| 90 |
+
hint_enabled: bool = True
|
| 91 |
+
show_on_mic_tap: bool = True
|
| 92 |
+
auto_hide_ms: int = 1400
|
| 93 |
+
hold_threshold_ms: int = 220
|
| 94 |
+
# Which entry modes this build accepts. Lets us A/B or disable a
|
| 95 |
+
# mode without a client change.
|
| 96 |
+
accepted_entry_modes: List[EntryMode] = Field(
|
| 97 |
+
default_factory=lambda: ["header_button", "long_press", "hint_tap"]
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
# ββ WebSocket event envelope ββββββββββββββββββββββββββββββββββββββββββ
|
| 102 |
+
|
| 103 |
+
class WsEvent(BaseModel):
|
| 104 |
+
"""Every message over the WS β both directions β wraps into this.
|
| 105 |
+
|
| 106 |
+
``seq`` is monotonic per session for events the server originates;
|
| 107 |
+
client-originated events don't need it (server ignores it if set).
|
| 108 |
+
"""
|
| 109 |
+
type: str
|
| 110 |
+
seq: Optional[int] = None
|
| 111 |
+
ts: Optional[int] = None
|
| 112 |
+
payload: Dict[str, Any] = Field(default_factory=dict)
|
backend/app/voice_call/policy.py
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Server-enforced policy for voice-call sessions.
|
| 3 |
+
|
| 4 |
+
Never trust the client; every cap here is enforced before a session row
|
| 5 |
+
is inserted or a WebSocket is accepted. Counters live in the DB via
|
| 6 |
+
``store`` so they survive process restarts and apply across workers.
|
| 7 |
+
"""
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
from . import store
|
| 11 |
+
from .config import VoiceCallConfig
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class PolicyError(Exception):
|
| 15 |
+
"""Raised when a caller violates a policy cap. Carries an HTTP status
|
| 16 |
+
and a short machine-readable code so the router can return a clean
|
| 17 |
+
JSON error without leaking internals."""
|
| 18 |
+
|
| 19 |
+
def __init__(self, http_status: int, code: str, detail: str) -> None:
|
| 20 |
+
super().__init__(detail)
|
| 21 |
+
self.http_status = http_status
|
| 22 |
+
self.code = code
|
| 23 |
+
self.detail = detail
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
def check_session_create(*, user_id: str, cfg: VoiceCallConfig) -> None:
|
| 27 |
+
"""Called before :func:`store.create_session`. Raises ``PolicyError``
|
| 28 |
+
on any violation.
|
| 29 |
+
|
| 30 |
+
Checks applied, in order of cheapness:
|
| 31 |
+
1. ``VOICE_CALL_ENABLED`` β feature gate.
|
| 32 |
+
2. Concurrent session cap (default 2). Stops the "infinite parallel
|
| 33 |
+
calls" abuse vector.
|
| 34 |
+
3. Rate limit on session creates (default 10 / 60 s). Stops script
|
| 35 |
+
kiddies from hammering ``/sessions``.
|
| 36 |
+
"""
|
| 37 |
+
if not cfg.enabled:
|
| 38 |
+
raise PolicyError(
|
| 39 |
+
http_status=404, code="voice_call_disabled",
|
| 40 |
+
detail="Voice call feature is not enabled on this server.",
|
| 41 |
+
)
|
| 42 |
+
|
| 43 |
+
active = store.count_active_sessions_for_user(user_id)
|
| 44 |
+
if active >= cfg.max_concurrent_per_user:
|
| 45 |
+
raise PolicyError(
|
| 46 |
+
http_status=409, code="too_many_active_sessions",
|
| 47 |
+
detail=(
|
| 48 |
+
f"You already have {active} active voice call(s). "
|
| 49 |
+
f"End one before starting a new one."
|
| 50 |
+
),
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
recent = store.recent_session_creates_for_user(
|
| 54 |
+
user_id, window_sec=60,
|
| 55 |
+
)
|
| 56 |
+
if recent >= cfg.session_create_rate_per_min:
|
| 57 |
+
raise PolicyError(
|
| 58 |
+
http_status=429, code="rate_limited",
|
| 59 |
+
detail=(
|
| 60 |
+
f"Too many voice call starts in the last minute "
|
| 61 |
+
f"(limit {cfg.session_create_rate_per_min})."
|
| 62 |
+
),
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def session_hard_cap_exceeded(session_row: dict, cfg: VoiceCallConfig) -> bool:
|
| 67 |
+
"""True if the session has exceeded its hard duration cap.
|
| 68 |
+
The WS orchestrator checks this every heartbeat to force ``ending``."""
|
| 69 |
+
started = int(session_row.get("started_at") or 0)
|
| 70 |
+
if started <= 0:
|
| 71 |
+
return False
|
| 72 |
+
now_ms = int(__import__("time").time() * 1000)
|
| 73 |
+
return (now_ms - started) >= cfg.max_duration_sec * 1000
|
backend/app/voice_call/router.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Public HTTP + WebSocket surface.
|
| 3 |
+
|
| 4 |
+
All endpoints live under ``/v1/voice-call/*``. The router is built
|
| 5 |
+
dynamically so that with ``VOICE_CALL_ENABLED=false`` it returns an
|
| 6 |
+
empty router β no endpoints, no 500s, no accidental exposure.
|
| 7 |
+
|
| 8 |
+
Mount from main.py ONCE, always behind the feature flag::
|
| 9 |
+
|
| 10 |
+
from .voice_call.router import build_router, build_ws_router
|
| 11 |
+
from .voice_call.config import load as _vc_load
|
| 12 |
+
_vc_cfg = _vc_load()
|
| 13 |
+
if _vc_cfg.enabled:
|
| 14 |
+
app.include_router(build_router(_vc_cfg))
|
| 15 |
+
app.include_router(build_ws_router(_vc_cfg))
|
| 16 |
+
"""
|
| 17 |
+
from __future__ import annotations
|
| 18 |
+
|
| 19 |
+
import logging
|
| 20 |
+
import os
|
| 21 |
+
from typing import Optional
|
| 22 |
+
|
| 23 |
+
from fastapi import (
|
| 24 |
+
APIRouter, Cookie, Depends, Header, HTTPException,
|
| 25 |
+
Query, WebSocket,
|
| 26 |
+
)
|
| 27 |
+
from fastapi.responses import JSONResponse
|
| 28 |
+
|
| 29 |
+
from . import service, store, ws as ws_mod
|
| 30 |
+
from .config import VoiceCallConfig
|
| 31 |
+
from .models import (
|
| 32 |
+
Capabilities, CreateSessionReq, CreateSessionResp,
|
| 33 |
+
EndSessionResp, HintsResp, Session,
|
| 34 |
+
)
|
| 35 |
+
from .policy import PolicyError
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
logger = logging.getLogger("voice_call.http")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
# ββ shared auth resolver (matches main._scoped_user_or_none) ββββββββββ
|
| 42 |
+
# Kept inline to avoid a circular import. Same semantics: single-user
|
| 43 |
+
# installs auto-resolve to the default user; multi-user installs require
|
| 44 |
+
# a bearer token.
|
| 45 |
+
|
| 46 |
+
def _resolve_user(
|
| 47 |
+
authorization: str,
|
| 48 |
+
homepilot_session: Optional[str],
|
| 49 |
+
) -> dict:
|
| 50 |
+
from ..users import (
|
| 51 |
+
ensure_users_tables, _validate_token,
|
| 52 |
+
get_or_create_default_user, count_users,
|
| 53 |
+
)
|
| 54 |
+
ensure_users_tables()
|
| 55 |
+
token = ""
|
| 56 |
+
if authorization and authorization.lower().startswith("bearer "):
|
| 57 |
+
token = authorization.split(" ", 1)[1].strip()
|
| 58 |
+
if not token and homepilot_session:
|
| 59 |
+
token = homepilot_session.strip()
|
| 60 |
+
user = _validate_token(token) if token else None
|
| 61 |
+
if user:
|
| 62 |
+
return user
|
| 63 |
+
if count_users() > 1:
|
| 64 |
+
raise HTTPException(status_code=401, detail="Authentication required")
|
| 65 |
+
return get_or_create_default_user()
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
# ββ helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 69 |
+
|
| 70 |
+
def _public_session(row: dict) -> Session:
|
| 71 |
+
"""Strip the resume_token out before the dict leaves the server."""
|
| 72 |
+
duration = None
|
| 73 |
+
if row.get("ended_at") and row.get("started_at"):
|
| 74 |
+
duration = int((int(row["ended_at"]) - int(row["started_at"])) / 1000)
|
| 75 |
+
return Session(
|
| 76 |
+
id=row["id"],
|
| 77 |
+
user_id=row["user_id"],
|
| 78 |
+
conversation_id=row.get("conversation_id"),
|
| 79 |
+
persona_id=row.get("persona_id"),
|
| 80 |
+
entry_mode=row["entry_mode"],
|
| 81 |
+
status=row["status"],
|
| 82 |
+
started_at=int(row["started_at"]),
|
| 83 |
+
ended_at=int(row["ended_at"]) if row.get("ended_at") else None,
|
| 84 |
+
ended_reason=row.get("ended_reason"),
|
| 85 |
+
duration_sec=duration,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def _ws_url_for(session_id: str) -> str:
|
| 90 |
+
"""The WS URL the client should open. Override via env for
|
| 91 |
+
containerized deployments where 127.0.0.1 is wrong."""
|
| 92 |
+
base = (
|
| 93 |
+
os.getenv("VOICE_CALL_PUBLIC_WS_BASE")
|
| 94 |
+
or "ws://127.0.0.1:8000"
|
| 95 |
+
).rstrip("/")
|
| 96 |
+
return f"{base}/v1/voice-call/ws/{session_id}"
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# ββ HTTP router ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 100 |
+
|
| 101 |
+
def build_router(cfg: VoiceCallConfig) -> APIRouter:
|
| 102 |
+
"""Return the REST sub-router. Empty router if feature off."""
|
| 103 |
+
r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"])
|
| 104 |
+
if not cfg.enabled:
|
| 105 |
+
return r # feature off β empty surface
|
| 106 |
+
|
| 107 |
+
@r.post("/sessions", response_model=CreateSessionResp, status_code=201)
|
| 108 |
+
def create_session(
|
| 109 |
+
body: CreateSessionReq,
|
| 110 |
+
authorization: str = Header(default=""),
|
| 111 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 112 |
+
):
|
| 113 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 114 |
+
try:
|
| 115 |
+
row = service.create_session(
|
| 116 |
+
user_id=user["id"],
|
| 117 |
+
conversation_id=body.conversation_id,
|
| 118 |
+
persona_id=body.persona_id,
|
| 119 |
+
entry_mode=body.entry_mode,
|
| 120 |
+
client_platform=body.device_info.platform,
|
| 121 |
+
app_version=body.device_info.app_version,
|
| 122 |
+
cfg=cfg,
|
| 123 |
+
)
|
| 124 |
+
except PolicyError as exc:
|
| 125 |
+
logger.warning(
|
| 126 |
+
"[call] create_session policy_error user=%s code=%s status=%s",
|
| 127 |
+
user["id"], exc.code, exc.http_status,
|
| 128 |
+
)
|
| 129 |
+
return JSONResponse(
|
| 130 |
+
status_code=exc.http_status,
|
| 131 |
+
content={"ok": False, "code": exc.code, "detail": exc.detail},
|
| 132 |
+
)
|
| 133 |
+
logger.info(
|
| 134 |
+
"[call] create_session ok user=%s sid=%s persona=%s conv=%s",
|
| 135 |
+
user["id"], row["id"], body.persona_id, body.conversation_id,
|
| 136 |
+
)
|
| 137 |
+
# Token is only returned in the create response β never again.
|
| 138 |
+
expires_at = int(row["created_at"]) + 15 * 60 * 1000
|
| 139 |
+
return CreateSessionResp(
|
| 140 |
+
session_id=row["id"],
|
| 141 |
+
ws_url=_ws_url_for(row["id"]),
|
| 142 |
+
resume_token=row["resume_token"],
|
| 143 |
+
expires_at=expires_at,
|
| 144 |
+
max_duration_sec=cfg.max_duration_sec,
|
| 145 |
+
capabilities=Capabilities(
|
| 146 |
+
interruptions=False,
|
| 147 |
+
# barge_in reflects the combined streaming + barge-in
|
| 148 |
+
# flag pair β it only works when the session actually
|
| 149 |
+
# streams, since there's nothing to cancel in unary.
|
| 150 |
+
barge_in=cfg.streaming_enabled and cfg.barge_in_enabled,
|
| 151 |
+
# transcript_live = streaming assistant partials.
|
| 152 |
+
transcript_live=cfg.streaming_enabled,
|
| 153 |
+
streaming=cfg.streaming_enabled,
|
| 154 |
+
),
|
| 155 |
+
)
|
| 156 |
+
|
| 157 |
+
@r.get("/sessions/{sid}", response_model=Session)
|
| 158 |
+
def get_session(
|
| 159 |
+
sid: str,
|
| 160 |
+
authorization: str = Header(default=""),
|
| 161 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 162 |
+
):
|
| 163 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 164 |
+
row = service.get(sid, user["id"])
|
| 165 |
+
if row is None:
|
| 166 |
+
# 404, never 403 β probe-safe.
|
| 167 |
+
logger.info("[call] get_session miss user=%s sid=%s", user["id"], sid)
|
| 168 |
+
raise HTTPException(status_code=404, detail="Session not found")
|
| 169 |
+
return _public_session(row)
|
| 170 |
+
|
| 171 |
+
@r.post("/sessions/{sid}/end", response_model=EndSessionResp)
|
| 172 |
+
def end_session_ep(
|
| 173 |
+
sid: str,
|
| 174 |
+
authorization: str = Header(default=""),
|
| 175 |
+
homepilot_session: Optional[str] = Cookie(default=None),
|
| 176 |
+
):
|
| 177 |
+
user = _resolve_user(authorization, homepilot_session)
|
| 178 |
+
row = service.end_session(sid=sid, user_id=user["id"])
|
| 179 |
+
if row is None:
|
| 180 |
+
logger.info("[call] end_session miss user=%s sid=%s", user["id"], sid)
|
| 181 |
+
raise HTTPException(status_code=404, detail="Session not found")
|
| 182 |
+
logger.info(
|
| 183 |
+
"[call] end_session ok user=%s sid=%s reason=%s",
|
| 184 |
+
user["id"], sid, row.get("ended_reason"),
|
| 185 |
+
)
|
| 186 |
+
public = _public_session(row)
|
| 187 |
+
return EndSessionResp(
|
| 188 |
+
session_id=public.id,
|
| 189 |
+
status=public.status,
|
| 190 |
+
ended_at=public.ended_at or int(row["updated_at"]),
|
| 191 |
+
ended_reason=public.ended_reason or "user_ended",
|
| 192 |
+
duration_sec=public.duration_sec or 0,
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
@r.get("/hints", response_model=HintsResp)
|
| 196 |
+
def hints():
|
| 197 |
+
# Public on purpose β the client needs this BEFORE it has a
|
| 198 |
+
# session. Still costs nothing to allow without auth.
|
| 199 |
+
return HintsResp()
|
| 200 |
+
|
| 201 |
+
return r
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
# ββ WS router ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 205 |
+
|
| 206 |
+
def build_ws_router(cfg: VoiceCallConfig) -> APIRouter:
|
| 207 |
+
r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"])
|
| 208 |
+
if not cfg.enabled or not cfg.websocket_enabled:
|
| 209 |
+
return r
|
| 210 |
+
|
| 211 |
+
@r.websocket("/ws/{session_id}")
|
| 212 |
+
async def voice_call_ws(
|
| 213 |
+
websocket: WebSocket,
|
| 214 |
+
session_id: str,
|
| 215 |
+
resume_token: Optional[str] = Query(default=None),
|
| 216 |
+
):
|
| 217 |
+
# Accept first so we can close with a clean error code if needed.
|
| 218 |
+
await websocket.accept()
|
| 219 |
+
await ws_mod.run_session_ws(
|
| 220 |
+
websocket, session_id, resume_token, cfg,
|
| 221 |
+
)
|
| 222 |
+
|
| 223 |
+
return r
|
backend/app/voice_call/service.py
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Pure-Python state machine for a voice-call session.
|
| 3 |
+
|
| 4 |
+
No HTTP or WebSocket imports here β the router/WS layers consume this
|
| 5 |
+
module. That separation lets the unit tests drive the state machine
|
| 6 |
+
end-to-end without spinning up FastAPI.
|
| 7 |
+
|
| 8 |
+
State flow (from the design doc, collapsed for MVP):
|
| 9 |
+
|
| 10 |
+
connecting β live β ending β ended
|
| 11 |
+
β β
|
| 12 |
+
β ββ interrupted (WS dropped, within resume window)
|
| 13 |
+
β β β
|
| 14 |
+
β β βββ live again (on resume)
|
| 15 |
+
β β βββ ended (resume window expired)
|
| 16 |
+
β
|
| 17 |
+
ββ ended (ws never opens)
|
| 18 |
+
"""
|
| 19 |
+
from __future__ import annotations
|
| 20 |
+
|
| 21 |
+
import time
|
| 22 |
+
from typing import Any, Dict, Optional
|
| 23 |
+
|
| 24 |
+
from . import store
|
| 25 |
+
from .config import VoiceCallConfig
|
| 26 |
+
from .policy import PolicyError, check_session_create
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
_TERMINAL_STATES = {"ended"}
|
| 30 |
+
_LIVE_STATES = {"connecting", "live", "interrupted"}
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
def _now_ms() -> int:
|
| 34 |
+
return int(time.time() * 1000)
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
# ββ create / end βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 38 |
+
|
| 39 |
+
def create_session(
|
| 40 |
+
*,
|
| 41 |
+
user_id: str,
|
| 42 |
+
conversation_id: Optional[str],
|
| 43 |
+
persona_id: Optional[str],
|
| 44 |
+
entry_mode: str,
|
| 45 |
+
client_platform: Optional[str],
|
| 46 |
+
app_version: Optional[str],
|
| 47 |
+
cfg: VoiceCallConfig,
|
| 48 |
+
) -> Dict[str, Any]:
|
| 49 |
+
"""Gate through policy, then insert a new session row.
|
| 50 |
+
|
| 51 |
+
The returned dict includes the ``resume_token`` which the router
|
| 52 |
+
will forward to the client. Callers MUST strip this from any
|
| 53 |
+
response that ends up in an observability log.
|
| 54 |
+
"""
|
| 55 |
+
check_session_create(user_id=user_id, cfg=cfg)
|
| 56 |
+
row = store.create_session(
|
| 57 |
+
user_id=user_id,
|
| 58 |
+
conversation_id=conversation_id,
|
| 59 |
+
persona_id=persona_id,
|
| 60 |
+
entry_mode=entry_mode,
|
| 61 |
+
client_platform=client_platform,
|
| 62 |
+
app_version=app_version,
|
| 63 |
+
)
|
| 64 |
+
store.append_event(
|
| 65 |
+
session_id=row["id"],
|
| 66 |
+
seq=0,
|
| 67 |
+
event_type="session.created",
|
| 68 |
+
payload={"entry_mode": entry_mode},
|
| 69 |
+
)
|
| 70 |
+
return row
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def end_session(
|
| 74 |
+
*,
|
| 75 |
+
sid: str,
|
| 76 |
+
user_id: str,
|
| 77 |
+
reason: str = "user_ended",
|
| 78 |
+
) -> Optional[Dict[str, Any]]:
|
| 79 |
+
"""Mark a session ``ended``. Idempotent: calling twice returns the
|
| 80 |
+
same row on the second call. Returns ``None`` if the session doesn't
|
| 81 |
+
exist OR doesn't belong to ``user_id`` (probe-safe)."""
|
| 82 |
+
row = store.get_session_for_owner(sid, user_id)
|
| 83 |
+
if row is None:
|
| 84 |
+
return None
|
| 85 |
+
if row["status"] in _TERMINAL_STATES:
|
| 86 |
+
return row # already ended; idempotent
|
| 87 |
+
updated = store.update_session_status(sid, "ended", ended_reason=reason)
|
| 88 |
+
if updated is not None:
|
| 89 |
+
store.append_event(
|
| 90 |
+
session_id=sid,
|
| 91 |
+
seq=_next_event_seq(sid),
|
| 92 |
+
event_type="session.ended",
|
| 93 |
+
payload={"reason": reason},
|
| 94 |
+
)
|
| 95 |
+
return updated
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def get(sid: str, user_id: str) -> Optional[Dict[str, Any]]:
|
| 99 |
+
"""Owner-bound getter. Mirrors :func:`store.get_session_for_owner`
|
| 100 |
+
so callers don't import store directly."""
|
| 101 |
+
return store.get_session_for_owner(sid, user_id)
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
# ββ internal state transitions (called from ws.py) βββββββββββββββββββ
|
| 105 |
+
|
| 106 |
+
def mark_live(sid: str) -> Optional[Dict[str, Any]]:
|
| 107 |
+
"""Transition from ``connecting`` β ``live``. Called when the WS
|
| 108 |
+
opens with a valid resume token."""
|
| 109 |
+
return store.update_session_status(sid, "live")
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def mark_interrupted(sid: str) -> Optional[Dict[str, Any]]:
|
| 113 |
+
return store.update_session_status(sid, "interrupted")
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
def mark_ended_by_timeout(sid: str, reason: str) -> Optional[Dict[str, Any]]:
|
| 117 |
+
return store.update_session_status(sid, "ended", ended_reason=reason)
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
# ββ sequence bookkeeping βββββββββββββββββββββββββββββββββββββββββββββ
|
| 121 |
+
|
| 122 |
+
# The server emits events with a monotonically-increasing ``seq``. We
|
| 123 |
+
# keep a tiny per-process cache to avoid a SELECT MAX() on every write.
|
| 124 |
+
# The cache is seeded on first use by reading the current max seq for
|
| 125 |
+
# the session β so it recovers cleanly across process restarts.
|
| 126 |
+
|
| 127 |
+
from threading import Lock
|
| 128 |
+
_seq_cache: Dict[str, int] = {}
|
| 129 |
+
_seq_lock = Lock()
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _next_event_seq(session_id: str) -> int:
|
| 133 |
+
with _seq_lock:
|
| 134 |
+
if session_id in _seq_cache:
|
| 135 |
+
_seq_cache[session_id] += 1
|
| 136 |
+
return _seq_cache[session_id]
|
| 137 |
+
events = store.list_events(session_id, limit=1000)
|
| 138 |
+
current_max = max((e["seq"] for e in events), default=-1)
|
| 139 |
+
_seq_cache[session_id] = current_max + 1
|
| 140 |
+
return _seq_cache[session_id]
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def _reset_for_tests() -> None:
|
| 144 |
+
with _seq_lock:
|
| 145 |
+
_seq_cache.clear()
|
backend/app/voice_call/store.py
ADDED
|
@@ -0,0 +1,293 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
SQLite access for voice_call.
|
| 3 |
+
|
| 4 |
+
Follows the same pattern as backend/app/storage.py and users.py:
|
| 5 |
+
direct sqlite3 calls, no ORM, per-call connections. Tables are created
|
| 6 |
+
lazily by :func:`ensure_schema` β no separate migration file, no side
|
| 7 |
+
effects on import. If the feature is disabled, this module is never
|
| 8 |
+
loaded and no tables are created.
|
| 9 |
+
"""
|
| 10 |
+
from __future__ import annotations
|
| 11 |
+
|
| 12 |
+
import json
|
| 13 |
+
import secrets
|
| 14 |
+
import sqlite3
|
| 15 |
+
import time
|
| 16 |
+
from typing import Any, Dict, List, Optional
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# Re-use the same DB path the rest of HomePilot uses.
|
| 20 |
+
def _db_path() -> str:
|
| 21 |
+
from ..storage import _get_db_path # lazy import, avoids cycles
|
| 22 |
+
return _get_db_path()
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def _conn() -> sqlite3.Connection:
|
| 26 |
+
con = sqlite3.connect(_db_path())
|
| 27 |
+
con.row_factory = sqlite3.Row
|
| 28 |
+
return con
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
_SCHEMA_READY = False
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def ensure_schema() -> None:
|
| 35 |
+
"""Create voice_call tables if they don't exist. Safe to call repeatedly."""
|
| 36 |
+
global _SCHEMA_READY
|
| 37 |
+
if _SCHEMA_READY:
|
| 38 |
+
return
|
| 39 |
+
con = _conn()
|
| 40 |
+
cur = con.cursor()
|
| 41 |
+
cur.executescript(
|
| 42 |
+
"""
|
| 43 |
+
CREATE TABLE IF NOT EXISTS voice_call_sessions (
|
| 44 |
+
id TEXT PRIMARY KEY,
|
| 45 |
+
user_id TEXT NOT NULL,
|
| 46 |
+
conversation_id TEXT,
|
| 47 |
+
persona_id TEXT,
|
| 48 |
+
entry_mode TEXT NOT NULL,
|
| 49 |
+
status TEXT NOT NULL,
|
| 50 |
+
resume_token TEXT NOT NULL,
|
| 51 |
+
started_at INTEGER NOT NULL,
|
| 52 |
+
ended_at INTEGER,
|
| 53 |
+
ended_reason TEXT,
|
| 54 |
+
client_platform TEXT,
|
| 55 |
+
app_version TEXT,
|
| 56 |
+
created_at INTEGER NOT NULL,
|
| 57 |
+
updated_at INTEGER NOT NULL
|
| 58 |
+
);
|
| 59 |
+
CREATE INDEX IF NOT EXISTS ix_vcs_user_started
|
| 60 |
+
ON voice_call_sessions (user_id, started_at DESC);
|
| 61 |
+
CREATE INDEX IF NOT EXISTS ix_vcs_user_status
|
| 62 |
+
ON voice_call_sessions (user_id, status);
|
| 63 |
+
|
| 64 |
+
CREATE TABLE IF NOT EXISTS voice_call_events (
|
| 65 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 66 |
+
session_id TEXT NOT NULL,
|
| 67 |
+
seq INTEGER NOT NULL,
|
| 68 |
+
event_type TEXT NOT NULL,
|
| 69 |
+
event_ts INTEGER NOT NULL,
|
| 70 |
+
payload TEXT NOT NULL DEFAULT '{}'
|
| 71 |
+
);
|
| 72 |
+
CREATE INDEX IF NOT EXISTS ix_vce_session_seq
|
| 73 |
+
ON voice_call_events (session_id, seq);
|
| 74 |
+
"""
|
| 75 |
+
)
|
| 76 |
+
con.commit()
|
| 77 |
+
con.close()
|
| 78 |
+
_SCHEMA_READY = True
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
# ββ session CRUD ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 82 |
+
|
| 83 |
+
def new_session_id() -> str:
|
| 84 |
+
return "vcs_" + secrets.token_urlsafe(16)
|
| 85 |
+
|
| 86 |
+
|
| 87 |
+
def new_resume_token() -> str:
|
| 88 |
+
return secrets.token_urlsafe(24)
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def _now_ms() -> int:
|
| 92 |
+
return int(time.time() * 1000)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def create_session(
|
| 96 |
+
*,
|
| 97 |
+
user_id: str,
|
| 98 |
+
conversation_id: Optional[str],
|
| 99 |
+
persona_id: Optional[str],
|
| 100 |
+
entry_mode: str,
|
| 101 |
+
client_platform: Optional[str],
|
| 102 |
+
app_version: Optional[str],
|
| 103 |
+
) -> Dict[str, Any]:
|
| 104 |
+
"""Insert a session row in the ``connecting`` state and return it."""
|
| 105 |
+
ensure_schema()
|
| 106 |
+
sid = new_session_id()
|
| 107 |
+
token = new_resume_token()
|
| 108 |
+
now = _now_ms()
|
| 109 |
+
con = _conn()
|
| 110 |
+
cur = con.cursor()
|
| 111 |
+
cur.execute(
|
| 112 |
+
"""
|
| 113 |
+
INSERT INTO voice_call_sessions
|
| 114 |
+
(id, user_id, conversation_id, persona_id, entry_mode, status,
|
| 115 |
+
resume_token, started_at, ended_at, ended_reason,
|
| 116 |
+
client_platform, app_version, created_at, updated_at)
|
| 117 |
+
VALUES
|
| 118 |
+
(?, ?, ?, ?, ?, 'connecting', ?, ?, NULL, NULL, ?, ?, ?, ?)
|
| 119 |
+
""",
|
| 120 |
+
(sid, user_id, conversation_id, persona_id, entry_mode, token,
|
| 121 |
+
now, client_platform, app_version, now, now),
|
| 122 |
+
)
|
| 123 |
+
con.commit()
|
| 124 |
+
con.close()
|
| 125 |
+
return _select_session(sid) # type: ignore[return-value]
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
|
| 129 |
+
d = {k: row[k] for k in row.keys()}
|
| 130 |
+
return d
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def _select_session(sid: str) -> Optional[Dict[str, Any]]:
|
| 134 |
+
ensure_schema()
|
| 135 |
+
con = _conn()
|
| 136 |
+
cur = con.cursor()
|
| 137 |
+
cur.execute("SELECT * FROM voice_call_sessions WHERE id = ?", (sid,))
|
| 138 |
+
row = cur.fetchone()
|
| 139 |
+
con.close()
|
| 140 |
+
return _row_to_dict(row) if row else None
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def get_session_for_owner(sid: str, user_id: str) -> Optional[Dict[str, Any]]:
|
| 144 |
+
"""Fetch a session ONLY if it belongs to ``user_id``.
|
| 145 |
+
|
| 146 |
+
Returns ``None`` for both "not found" and "wrong owner" so callers
|
| 147 |
+
cannot probe session ids they don't own. Matches the pattern used
|
| 148 |
+
by studio render jobs.
|
| 149 |
+
"""
|
| 150 |
+
row = _select_session(sid)
|
| 151 |
+
if row and row["user_id"] == user_id:
|
| 152 |
+
return row
|
| 153 |
+
return None
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def count_active_sessions_for_user(user_id: str) -> int:
|
| 157 |
+
"""Count sessions that are still 'live' or 'connecting' or 'interrupted'."""
|
| 158 |
+
ensure_schema()
|
| 159 |
+
con = _conn()
|
| 160 |
+
cur = con.cursor()
|
| 161 |
+
cur.execute(
|
| 162 |
+
"""
|
| 163 |
+
SELECT COUNT(*) AS n FROM voice_call_sessions
|
| 164 |
+
WHERE user_id = ?
|
| 165 |
+
AND status IN ('connecting', 'live', 'interrupted')
|
| 166 |
+
""",
|
| 167 |
+
(user_id,),
|
| 168 |
+
)
|
| 169 |
+
n = int(cur.fetchone()["n"])
|
| 170 |
+
con.close()
|
| 171 |
+
return n
|
| 172 |
+
|
| 173 |
+
|
| 174 |
+
def recent_session_creates_for_user(user_id: str, window_sec: int) -> int:
|
| 175 |
+
"""Session creates by ``user_id`` in the last ``window_sec`` seconds."""
|
| 176 |
+
ensure_schema()
|
| 177 |
+
con = _conn()
|
| 178 |
+
cur = con.cursor()
|
| 179 |
+
since = _now_ms() - int(window_sec * 1000)
|
| 180 |
+
cur.execute(
|
| 181 |
+
"""
|
| 182 |
+
SELECT COUNT(*) AS n FROM voice_call_sessions
|
| 183 |
+
WHERE user_id = ? AND created_at >= ?
|
| 184 |
+
""",
|
| 185 |
+
(user_id, since),
|
| 186 |
+
)
|
| 187 |
+
n = int(cur.fetchone()["n"])
|
| 188 |
+
con.close()
|
| 189 |
+
return n
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def update_session_status(
|
| 193 |
+
sid: str,
|
| 194 |
+
status: str,
|
| 195 |
+
*,
|
| 196 |
+
ended_reason: Optional[str] = None,
|
| 197 |
+
) -> Optional[Dict[str, Any]]:
|
| 198 |
+
ensure_schema()
|
| 199 |
+
now = _now_ms()
|
| 200 |
+
con = _conn()
|
| 201 |
+
cur = con.cursor()
|
| 202 |
+
if status == "ended":
|
| 203 |
+
cur.execute(
|
| 204 |
+
"""
|
| 205 |
+
UPDATE voice_call_sessions
|
| 206 |
+
SET status = 'ended', ended_at = ?, ended_reason = ?, updated_at = ?
|
| 207 |
+
WHERE id = ?
|
| 208 |
+
""",
|
| 209 |
+
(now, ended_reason or "user_ended", now, sid),
|
| 210 |
+
)
|
| 211 |
+
else:
|
| 212 |
+
cur.execute(
|
| 213 |
+
"""
|
| 214 |
+
UPDATE voice_call_sessions
|
| 215 |
+
SET status = ?, updated_at = ?
|
| 216 |
+
WHERE id = ?
|
| 217 |
+
""",
|
| 218 |
+
(status, now, sid),
|
| 219 |
+
)
|
| 220 |
+
con.commit()
|
| 221 |
+
con.close()
|
| 222 |
+
return _select_session(sid)
|
| 223 |
+
|
| 224 |
+
|
| 225 |
+
# ββ events ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 226 |
+
|
| 227 |
+
def append_event(
|
| 228 |
+
*,
|
| 229 |
+
session_id: str,
|
| 230 |
+
seq: int,
|
| 231 |
+
event_type: str,
|
| 232 |
+
payload: Optional[Dict[str, Any]] = None,
|
| 233 |
+
) -> None:
|
| 234 |
+
"""Append a structured event row. Payload defaults to {} β DO NOT
|
| 235 |
+
pass raw audio or full user text here unless privacy flag allows it."""
|
| 236 |
+
ensure_schema()
|
| 237 |
+
con = _conn()
|
| 238 |
+
cur = con.cursor()
|
| 239 |
+
cur.execute(
|
| 240 |
+
"""
|
| 241 |
+
INSERT INTO voice_call_events (session_id, seq, event_type, event_ts, payload)
|
| 242 |
+
VALUES (?, ?, ?, ?, ?)
|
| 243 |
+
""",
|
| 244 |
+
(
|
| 245 |
+
session_id,
|
| 246 |
+
int(seq),
|
| 247 |
+
event_type,
|
| 248 |
+
_now_ms(),
|
| 249 |
+
json.dumps(payload or {}, separators=(",", ":")),
|
| 250 |
+
),
|
| 251 |
+
)
|
| 252 |
+
con.commit()
|
| 253 |
+
con.close()
|
| 254 |
+
|
| 255 |
+
|
| 256 |
+
def list_events(session_id: str, limit: int = 500) -> List[Dict[str, Any]]:
|
| 257 |
+
ensure_schema()
|
| 258 |
+
con = _conn()
|
| 259 |
+
cur = con.cursor()
|
| 260 |
+
cur.execute(
|
| 261 |
+
"""
|
| 262 |
+
SELECT session_id, seq, event_type, event_ts, payload
|
| 263 |
+
FROM voice_call_events
|
| 264 |
+
WHERE session_id = ?
|
| 265 |
+
ORDER BY seq ASC
|
| 266 |
+
LIMIT ?
|
| 267 |
+
""",
|
| 268 |
+
(session_id, int(limit)),
|
| 269 |
+
)
|
| 270 |
+
rows = cur.fetchall()
|
| 271 |
+
con.close()
|
| 272 |
+
out = []
|
| 273 |
+
for r in rows:
|
| 274 |
+
d = _row_to_dict(r)
|
| 275 |
+
try:
|
| 276 |
+
d["payload"] = json.loads(d.get("payload") or "{}")
|
| 277 |
+
except Exception:
|
| 278 |
+
d["payload"] = {}
|
| 279 |
+
out.append(d)
|
| 280 |
+
return out
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
# ββ test helper βββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 284 |
+
|
| 285 |
+
def _reset_for_tests() -> None:
|
| 286 |
+
"""DELETE every voice_call row. Only used by pytest fixtures."""
|
| 287 |
+
ensure_schema()
|
| 288 |
+
con = _conn()
|
| 289 |
+
cur = con.cursor()
|
| 290 |
+
cur.execute("DELETE FROM voice_call_events")
|
| 291 |
+
cur.execute("DELETE FROM voice_call_sessions")
|
| 292 |
+
con.commit()
|
| 293 |
+
con.close()
|
backend/app/voice_call/turn.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Turn bridge β send a ``transcript.final`` from the client to the
|
| 3 |
+
existing ``/v1/chat/completions`` endpoint and get back an assistant
|
| 4 |
+
reply (also final, no deltas).
|
| 5 |
+
|
| 6 |
+
Implementation note
|
| 7 |
+
-------------------
|
| 8 |
+
The review confirmed the compat endpoint currently returns 501 on
|
| 9 |
+
``stream=true`` (openai_compat_endpoint.py:477), so MVP M1 is final-
|
| 10 |
+
only. We invoke the endpoint via an internal loopback HTTP call; that
|
| 11 |
+
buys:
|
| 12 |
+
|
| 13 |
+
* zero import-time coupling to the chat module,
|
| 14 |
+
* the same auth / memory / persona context the normal chat path uses,
|
| 15 |
+
* a natural seam for the V2 realtime-provider adapter (swap this
|
| 16 |
+
function, everything else stays).
|
| 17 |
+
|
| 18 |
+
When the feature flag is off this module is never imported.
|
| 19 |
+
"""
|
| 20 |
+
from __future__ import annotations
|
| 21 |
+
|
| 22 |
+
import os
|
| 23 |
+
from typing import Any, Dict, List, Optional
|
| 24 |
+
|
| 25 |
+
import httpx
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def _local_backend_url() -> str:
|
| 29 |
+
"""Where to reach our own FastAPI server over loopback. Honors an
|
| 30 |
+
override for containerized / split-process deployments."""
|
| 31 |
+
return (
|
| 32 |
+
os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL")
|
| 33 |
+
or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL")
|
| 34 |
+
or "http://127.0.0.1:8000"
|
| 35 |
+
).rstrip("/")
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "30"))
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
async def run_turn(
|
| 42 |
+
*,
|
| 43 |
+
user_text: str,
|
| 44 |
+
model: str,
|
| 45 |
+
auth_bearer: Optional[str] = None,
|
| 46 |
+
system_prompt: Optional[str] = None,
|
| 47 |
+
additional_system: Optional[str] = None,
|
| 48 |
+
history: Optional[List[Dict[str, str]]] = None,
|
| 49 |
+
extra_headers: Optional[Dict[str, str]] = None,
|
| 50 |
+
) -> str:
|
| 51 |
+
"""Run a single assistant turn via the existing chat endpoint.
|
| 52 |
+
|
| 53 |
+
Parameters
|
| 54 |
+
----------
|
| 55 |
+
user_text:
|
| 56 |
+
The final STT transcript from the client.
|
| 57 |
+
model:
|
| 58 |
+
The model id. ``persona:<project_id>`` / ``personality:<id>`` /
|
| 59 |
+
plain ollama model name β whatever the existing chat endpoint
|
| 60 |
+
accepts.
|
| 61 |
+
auth_bearer:
|
| 62 |
+
The caller's bearer token, forwarded verbatim so the chat
|
| 63 |
+
endpoint sees the same user identity as the WS session owner.
|
| 64 |
+
system_prompt / history:
|
| 65 |
+
Optional pre-built context. Left empty the endpoint will attach
|
| 66 |
+
its own persona / memory context based on the model.
|
| 67 |
+
additional_system:
|
| 68 |
+
Optional extra system message appended AFTER ``system_prompt``
|
| 69 |
+
(or on its own if no ``system_prompt`` is provided). Additive
|
| 70 |
+
only β used by ``persona_call`` to inject a per-turn phone-call
|
| 71 |
+
suffix WITHOUT touching the persona's own system prompt that
|
| 72 |
+
the compat endpoint already injects for ``persona:*`` models.
|
| 73 |
+
|
| 74 |
+
Returns
|
| 75 |
+
-------
|
| 76 |
+
The assistant's reply text (the ``choices[0].message.content`` from
|
| 77 |
+
the chat completion response). Raises on non-2xx responses so the
|
| 78 |
+
caller can surface an ``error`` event over the WS.
|
| 79 |
+
"""
|
| 80 |
+
messages: List[Dict[str, str]] = []
|
| 81 |
+
if system_prompt:
|
| 82 |
+
messages.append({"role": "system", "content": system_prompt})
|
| 83 |
+
if additional_system:
|
| 84 |
+
messages.append({"role": "system", "content": additional_system})
|
| 85 |
+
if history:
|
| 86 |
+
messages.extend(history)
|
| 87 |
+
messages.append({"role": "user", "content": user_text})
|
| 88 |
+
|
| 89 |
+
headers: Dict[str, str] = {"Content-Type": "application/json"}
|
| 90 |
+
if auth_bearer:
|
| 91 |
+
headers["Authorization"] = f"Bearer {auth_bearer}"
|
| 92 |
+
if extra_headers:
|
| 93 |
+
headers.update(extra_headers)
|
| 94 |
+
|
| 95 |
+
payload = {
|
| 96 |
+
"model": model,
|
| 97 |
+
"messages": messages,
|
| 98 |
+
# Final-only β must stay False at MVP. The compat endpoint
|
| 99 |
+
# returns 501 on streaming requests.
|
| 100 |
+
"stream": False,
|
| 101 |
+
# Keep response tight for voice. Long prose makes for bad audio.
|
| 102 |
+
"max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")),
|
| 103 |
+
"temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")),
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
url = f"{_local_backend_url()}/v1/chat/completions"
|
| 107 |
+
async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client:
|
| 108 |
+
r = await client.post(url, json=payload, headers=headers)
|
| 109 |
+
# Let the caller see the HTTP body on error β useful for debugging.
|
| 110 |
+
if r.status_code >= 400:
|
| 111 |
+
raise RuntimeError(
|
| 112 |
+
f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}"
|
| 113 |
+
)
|
| 114 |
+
data = r.json()
|
| 115 |
+
try:
|
| 116 |
+
return (
|
| 117 |
+
data["choices"][0]["message"]["content"]
|
| 118 |
+
if data.get("choices")
|
| 119 |
+
else ""
|
| 120 |
+
)
|
| 121 |
+
except (KeyError, IndexError, TypeError):
|
| 122 |
+
raise RuntimeError(
|
| 123 |
+
"chat endpoint returned an unexpected body shape: "
|
| 124 |
+
+ str(data)[:400]
|
| 125 |
+
)
|
backend/app/voice_call/turn_stream.py
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Streaming turn runner β async-generator peer of :func:`turn.run_turn`.
|
| 3 |
+
|
| 4 |
+
Yields assistant text chunks from the chat provider as they arrive so
|
| 5 |
+
the WebSocket handler can emit ``assistant.partial`` envelopes without
|
| 6 |
+
waiting for the full reply. Drops cleanly on barge-in cancel.
|
| 7 |
+
|
| 8 |
+
Contract is documented in
|
| 9 |
+
``docs/analysis/voice-call-streaming-design.md`` Β§ 4.1 and Β§ 4.2.
|
| 10 |
+
|
| 11 |
+
Two execution paths, selected at runtime:
|
| 12 |
+
|
| 13 |
+
1. **Native streaming** β POST to the compat chat endpoint with
|
| 14 |
+
``stream: true`` and consume the SSE / NDJSON response. This is
|
| 15 |
+
the real Phase 2 path and delivers sub-500 ms first-audio when
|
| 16 |
+
the provider supports it.
|
| 17 |
+
|
| 18 |
+
2. **Chunked-unary fallback** β if the compat endpoint returns 501
|
| 19 |
+
(current default, see ``turn.py`` comment) OR the provider
|
| 20 |
+
doesn't support streaming, we call the unary endpoint, receive
|
| 21 |
+
the full reply, and emit it in clause-sized chunks with a tiny
|
| 22 |
+
delay so the downstream plumbing (``assistant.partial`` β
|
| 23 |
+
``streamTts.appendDelta``) still exercises correctly.
|
| 24 |
+
|
| 25 |
+
The fallback yields no latency improvement β its value is keeping
|
| 26 |
+
the envelope contract and the client-side streaming TTS alive even
|
| 27 |
+
while the compat endpoint hasn't flipped on streaming. When it does,
|
| 28 |
+
this module starts getting real deltas automatically.
|
| 29 |
+
|
| 30 |
+
The persona_call suffix, persona resolution, and auth are all handled
|
| 31 |
+
by ``ws.py`` / the compat endpoint exactly as today. This module
|
| 32 |
+
receives the already-resolved ``user_text`` + ``model`` + headers
|
| 33 |
+
and does nothing with the persona layer directly.
|
| 34 |
+
"""
|
| 35 |
+
from __future__ import annotations
|
| 36 |
+
|
| 37 |
+
import asyncio
|
| 38 |
+
import json
|
| 39 |
+
import os
|
| 40 |
+
import re
|
| 41 |
+
from typing import AsyncIterator, Dict, List, Optional
|
| 42 |
+
|
| 43 |
+
import httpx
|
| 44 |
+
|
| 45 |
+
from .barge_in import BargeInToken
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def _local_backend_url() -> str:
|
| 49 |
+
return (
|
| 50 |
+
os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL")
|
| 51 |
+
or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL")
|
| 52 |
+
or "http://127.0.0.1:8000"
|
| 53 |
+
).rstrip("/")
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "60"))
|
| 57 |
+
# Chunk pacing for the fallback path. Kept deliberately small so the
|
| 58 |
+
# fallback "feels like" streaming without flooding the client.
|
| 59 |
+
_FALLBACK_INTER_CHUNK_MS = int(os.getenv("VOICE_CALL_FALLBACK_CHUNK_MS", "35"))
|
| 60 |
+
|
| 61 |
+
# Simple clause splitter used only by the fallback path. Deliberately
|
| 62 |
+
# naive: split on sentence-enders first, then on commas if a sentence
|
| 63 |
+
# is too long for one chunk.
|
| 64 |
+
_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+")
|
| 65 |
+
_CHUNK_MAX_CHARS = 90
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def _split_for_chunking(text: str) -> List[str]:
|
| 69 |
+
if not text:
|
| 70 |
+
return []
|
| 71 |
+
parts = [p for p in _SENTENCE_SPLIT.split(text) if p]
|
| 72 |
+
out: List[str] = []
|
| 73 |
+
for part in parts:
|
| 74 |
+
if len(part) <= _CHUNK_MAX_CHARS:
|
| 75 |
+
out.append(part)
|
| 76 |
+
continue
|
| 77 |
+
# Long sentence β split on commas, rejoining short pieces so
|
| 78 |
+
# we don't over-chunk.
|
| 79 |
+
comma_bits = [b.strip() for b in part.split(",") if b.strip()]
|
| 80 |
+
buf = ""
|
| 81 |
+
for bit in comma_bits:
|
| 82 |
+
if len(buf) + len(bit) + 2 <= _CHUNK_MAX_CHARS:
|
| 83 |
+
buf = f"{buf}, {bit}" if buf else bit
|
| 84 |
+
else:
|
| 85 |
+
if buf:
|
| 86 |
+
out.append(buf)
|
| 87 |
+
buf = bit
|
| 88 |
+
if buf:
|
| 89 |
+
out.append(buf)
|
| 90 |
+
# Re-attach trailing whitespace so concatenation preserves spacing.
|
| 91 |
+
return [f"{p} " if not p.endswith(" ") else p for p in out]
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _extract_delta_from_sse_line(line: str) -> Optional[str]:
|
| 95 |
+
"""Parse one SSE / NDJSON frame from a streaming chat endpoint.
|
| 96 |
+
|
| 97 |
+
Supports both the OpenAI-style ``data: {...}`` SSE format and
|
| 98 |
+
Ollama's raw NDJSON (``{"message": {...}}`` per line).
|
| 99 |
+
"""
|
| 100 |
+
line = line.strip()
|
| 101 |
+
if not line:
|
| 102 |
+
return None
|
| 103 |
+
if line.startswith("data:"):
|
| 104 |
+
line = line[5:].strip()
|
| 105 |
+
if line == "[DONE]":
|
| 106 |
+
return None
|
| 107 |
+
try:
|
| 108 |
+
obj = json.loads(line)
|
| 109 |
+
except json.JSONDecodeError:
|
| 110 |
+
return None
|
| 111 |
+
# OpenAI streaming: choices[0].delta.content
|
| 112 |
+
try:
|
| 113 |
+
choices = obj.get("choices") or []
|
| 114 |
+
if choices:
|
| 115 |
+
delta = (choices[0] or {}).get("delta") or {}
|
| 116 |
+
content = delta.get("content")
|
| 117 |
+
if isinstance(content, str) and content:
|
| 118 |
+
return content
|
| 119 |
+
except (AttributeError, TypeError):
|
| 120 |
+
pass
|
| 121 |
+
# Ollama streaming: message.content
|
| 122 |
+
try:
|
| 123 |
+
msg = obj.get("message") or {}
|
| 124 |
+
content = msg.get("content")
|
| 125 |
+
if isinstance(content, str) and content:
|
| 126 |
+
return content
|
| 127 |
+
except (AttributeError, TypeError):
|
| 128 |
+
pass
|
| 129 |
+
return None
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
async def _yield_native_stream(
|
| 133 |
+
client: httpx.AsyncClient,
|
| 134 |
+
url: str,
|
| 135 |
+
payload: Dict,
|
| 136 |
+
headers: Dict[str, str],
|
| 137 |
+
cancel_token: BargeInToken,
|
| 138 |
+
) -> AsyncIterator[str]:
|
| 139 |
+
"""Native provider streaming path β returns an async iterator of
|
| 140 |
+
deltas. Caller handles 501 / 4xx / 5xx by catching the upstream
|
| 141 |
+
status before entering this coroutine."""
|
| 142 |
+
async with client.stream("POST", url, json=payload, headers=headers) as r:
|
| 143 |
+
if r.status_code == 501:
|
| 144 |
+
# Sentinel for "provider doesn't support streaming yet".
|
| 145 |
+
# Caller will fall back to chunked-unary.
|
| 146 |
+
await r.aread()
|
| 147 |
+
raise _StreamingUnsupported()
|
| 148 |
+
if r.status_code >= 400:
|
| 149 |
+
body = (await r.aread()).decode("utf-8", errors="replace")
|
| 150 |
+
raise RuntimeError(
|
| 151 |
+
f"chat stream {url} returned {r.status_code}: {body[:400]}"
|
| 152 |
+
)
|
| 153 |
+
async for line in r.aiter_lines():
|
| 154 |
+
if cancel_token.is_cancelled():
|
| 155 |
+
return
|
| 156 |
+
delta = _extract_delta_from_sse_line(line)
|
| 157 |
+
if delta:
|
| 158 |
+
yield delta
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
class _StreamingUnsupported(Exception):
|
| 162 |
+
"""Raised when the provider returns 501 on ``stream: true`` so the
|
| 163 |
+
caller can route to the chunked-unary fallback without conflating
|
| 164 |
+
it with an ordinary upstream error."""
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
async def _yield_chunked_unary(
|
| 168 |
+
client: httpx.AsyncClient,
|
| 169 |
+
url: str,
|
| 170 |
+
payload: Dict,
|
| 171 |
+
headers: Dict[str, str],
|
| 172 |
+
cancel_token: BargeInToken,
|
| 173 |
+
) -> AsyncIterator[str]:
|
| 174 |
+
"""Fallback path β full unary call, then chunk the reply."""
|
| 175 |
+
unary_payload = dict(payload, stream=False)
|
| 176 |
+
r = await client.post(url, json=unary_payload, headers=headers)
|
| 177 |
+
if r.status_code >= 400:
|
| 178 |
+
raise RuntimeError(
|
| 179 |
+
f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}"
|
| 180 |
+
)
|
| 181 |
+
data = r.json()
|
| 182 |
+
try:
|
| 183 |
+
reply = (
|
| 184 |
+
data["choices"][0]["message"]["content"]
|
| 185 |
+
if data.get("choices")
|
| 186 |
+
else ""
|
| 187 |
+
)
|
| 188 |
+
except (KeyError, IndexError, TypeError):
|
| 189 |
+
raise RuntimeError(
|
| 190 |
+
f"chat endpoint returned an unexpected body shape: {str(data)[:400]}"
|
| 191 |
+
)
|
| 192 |
+
for chunk in _split_for_chunking(reply):
|
| 193 |
+
if cancel_token.is_cancelled():
|
| 194 |
+
return
|
| 195 |
+
yield chunk
|
| 196 |
+
# Small delay so the client's streaming TTS has a beat to
|
| 197 |
+
# synth + play before the next chunk arrives. Keeps the
|
| 198 |
+
# fallback from flooding the WS.
|
| 199 |
+
await asyncio.sleep(_FALLBACK_INTER_CHUNK_MS / 1000.0)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
async def run_turn_streaming(
|
| 203 |
+
*,
|
| 204 |
+
user_text: str,
|
| 205 |
+
model: str,
|
| 206 |
+
auth_bearer: Optional[str] = None,
|
| 207 |
+
system_prompt: Optional[str] = None,
|
| 208 |
+
additional_system: Optional[str] = None,
|
| 209 |
+
history: Optional[List[Dict[str, str]]] = None,
|
| 210 |
+
extra_headers: Optional[Dict[str, str]] = None,
|
| 211 |
+
cancel_token: BargeInToken,
|
| 212 |
+
) -> AsyncIterator[str]:
|
| 213 |
+
"""Yield assistant text chunks for the given user turn.
|
| 214 |
+
|
| 215 |
+
Exits cleanly (no exception) when ``cancel_token`` is tripped by
|
| 216 |
+
a barge-in. The caller is responsible for emitting the final
|
| 217 |
+
``assistant.turn_end`` envelope regardless of exit reason.
|
| 218 |
+
"""
|
| 219 |
+
messages: List[Dict[str, str]] = []
|
| 220 |
+
if system_prompt:
|
| 221 |
+
messages.append({"role": "system", "content": system_prompt})
|
| 222 |
+
if additional_system:
|
| 223 |
+
messages.append({"role": "system", "content": additional_system})
|
| 224 |
+
if history:
|
| 225 |
+
messages.extend(history)
|
| 226 |
+
messages.append({"role": "user", "content": user_text})
|
| 227 |
+
|
| 228 |
+
headers: Dict[str, str] = {"Content-Type": "application/json"}
|
| 229 |
+
if auth_bearer:
|
| 230 |
+
headers["Authorization"] = f"Bearer {auth_bearer}"
|
| 231 |
+
if extra_headers:
|
| 232 |
+
headers.update(extra_headers)
|
| 233 |
+
|
| 234 |
+
payload = {
|
| 235 |
+
"model": model,
|
| 236 |
+
"messages": messages,
|
| 237 |
+
"stream": True,
|
| 238 |
+
"max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")),
|
| 239 |
+
"temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")),
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
url = f"{_local_backend_url()}/v1/chat/completions"
|
| 243 |
+
|
| 244 |
+
async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client:
|
| 245 |
+
# Try native streaming first.
|
| 246 |
+
try:
|
| 247 |
+
async for delta in _yield_native_stream(
|
| 248 |
+
client, url, payload, headers, cancel_token,
|
| 249 |
+
):
|
| 250 |
+
yield delta
|
| 251 |
+
return
|
| 252 |
+
except _StreamingUnsupported:
|
| 253 |
+
# Compat endpoint currently returns 501 on stream=true;
|
| 254 |
+
# fall back to unary + chunker. Covered by test.
|
| 255 |
+
pass
|
| 256 |
+
async for delta in _yield_chunked_unary(
|
| 257 |
+
client, url, payload, headers, cancel_token,
|
| 258 |
+
):
|
| 259 |
+
yield delta
|
backend/app/voice_call/ws.py
ADDED
|
@@ -0,0 +1,605 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
WebSocket orchestrator for voice-call sessions.
|
| 3 |
+
|
| 4 |
+
Transport
|
| 5 |
+
---------
|
| 6 |
+
Path: ``/v1/voice-call/ws/{session_id}``
|
| 7 |
+
Query param: ``resume_token=<token issued at POST /sessions>``
|
| 8 |
+
(headers don't round-trip cleanly from browsers; a
|
| 9 |
+
short-lived query token is the standard WS-auth trick)
|
| 10 |
+
|
| 11 |
+
Protocol
|
| 12 |
+
--------
|
| 13 |
+
Server and client exchange JSON envelopes::
|
| 14 |
+
|
| 15 |
+
{ "type": "<event>", "seq": <int?>, "ts": <ms>, "payload": { ... } }
|
| 16 |
+
|
| 17 |
+
Client β server (each is harmless if the server can't handle it β
|
| 18 |
+
forward-compatible by design):
|
| 19 |
+
ui.state { muted, speaker_on, backgrounded }
|
| 20 |
+
transcript.final { text, lang? }
|
| 21 |
+
call.control { action: "end" | "ping" }
|
| 22 |
+
|
| 23 |
+
Server β client:
|
| 24 |
+
call.state { status, since }
|
| 25 |
+
transcript.final { role: "assistant", text }
|
| 26 |
+
error { code, message }
|
| 27 |
+
pong { ts }
|
| 28 |
+
safety.notice (forwarded from upstream providers, reserved)
|
| 29 |
+
|
| 30 |
+
Out of scope for MVP (per review): transcript.partial (requires chat
|
| 31 |
+
endpoint streaming, which returns 501 today), raw audio frames, server-
|
| 32 |
+
side barge-in.
|
| 33 |
+
"""
|
| 34 |
+
from __future__ import annotations
|
| 35 |
+
|
| 36 |
+
import asyncio
|
| 37 |
+
import json
|
| 38 |
+
import logging
|
| 39 |
+
import time
|
| 40 |
+
import uuid
|
| 41 |
+
from typing import Any, Dict, Optional
|
| 42 |
+
|
| 43 |
+
from fastapi import WebSocket, WebSocketDisconnect, status
|
| 44 |
+
|
| 45 |
+
from . import barge_in as _barge_in
|
| 46 |
+
from . import service, store, turn, turn_stream
|
| 47 |
+
from .config import VoiceCallConfig
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
logger = logging.getLogger("voice_call.ws")
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def _clog(event: str, **kwargs: Any) -> None:
|
| 54 |
+
"""Structured ``[call]`` log line β mirrors the frontend's clog()
|
| 55 |
+
format so a single grep surfaces the whole round-trip. One
|
| 56 |
+
``logger.info`` per inflection point; fields stay flat so
|
| 57 |
+
downstream log-shippers don't have to deserialize.
|
| 58 |
+
|
| 59 |
+
Used at: session accept (open), call.state live (ready), opener
|
| 60 |
+
greeting (greet), user transcript in (turn_user_in), assistant
|
| 61 |
+
reply out (turn_assistant_out), cooperative cancel (barge_in),
|
| 62 |
+
user-ended (hangup), WS teardown (close)."""
|
| 63 |
+
logger.info("[call] %s %s", event, json.dumps(kwargs, separators=(",", ":"), default=str))
|
| 64 |
+
|
| 65 |
+
HEARTBEAT_INTERVAL_SEC = 10.0
|
| 66 |
+
|
| 67 |
+
# Protocol sentinels β reserved strings the frontend may send as a
|
| 68 |
+
# ``transcript.final`` payload to nudge backend state without asking
|
| 69 |
+
# the LLM to react. We drop these on the floor before the turn runner
|
| 70 |
+
# sees them; otherwise the model treats them as literal user speech
|
| 71 |
+
# and produces a confused reply (the classic "I'm not sure what
|
| 72 |
+
# [phone-call-open] means" double-greet). Add new sentinels here only
|
| 73 |
+
# when both sides have landed.
|
| 74 |
+
_PROTOCOL_SENTINELS = frozenset({
|
| 75 |
+
"[phone-call-open]",
|
| 76 |
+
"[phone-call-close]",
|
| 77 |
+
})
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def _now_ms() -> int:
|
| 81 |
+
return int(time.time() * 1000)
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
async def _send(ws: WebSocket, type_: str, payload: Dict[str, Any], *,
|
| 85 |
+
session_id: str) -> None:
|
| 86 |
+
"""Send a serverβclient event with a monotonic seq number."""
|
| 87 |
+
seq = service._next_event_seq(session_id)
|
| 88 |
+
env = {"type": type_, "seq": seq, "ts": _now_ms(), "payload": payload}
|
| 89 |
+
try:
|
| 90 |
+
await ws.send_text(json.dumps(env, separators=(",", ":")))
|
| 91 |
+
except Exception:
|
| 92 |
+
# WebSocket already closed; caller will clean up.
|
| 93 |
+
raise
|
| 94 |
+
# Only persist the *type* for privacy unless the user has explicitly
|
| 95 |
+
# opted into transcript storage.
|
| 96 |
+
store.append_event(
|
| 97 |
+
session_id=session_id, seq=seq, event_type=type_,
|
| 98 |
+
payload={}, # intentionally empty
|
| 99 |
+
)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
async def _send_error(ws: WebSocket, session_id: str, code: str, msg: str) -> None:
|
| 103 |
+
try:
|
| 104 |
+
await _send(ws, "error",
|
| 105 |
+
{"code": code, "message": msg},
|
| 106 |
+
session_id=session_id)
|
| 107 |
+
except Exception:
|
| 108 |
+
pass # best-effort
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
async def _resolve_session_for_ws(
|
| 112 |
+
ws: WebSocket, session_id: str, resume_token: Optional[str],
|
| 113 |
+
cfg: VoiceCallConfig,
|
| 114 |
+
) -> Optional[Dict[str, Any]]:
|
| 115 |
+
"""Validate the handshake. Closes the socket with a clean code if
|
| 116 |
+
anything is off."""
|
| 117 |
+
if not cfg.websocket_enabled:
|
| 118 |
+
await ws.close(code=status.WS_1008_POLICY_VIOLATION,
|
| 119 |
+
reason="websocket-disabled")
|
| 120 |
+
return None
|
| 121 |
+
row = store._select_session(session_id)
|
| 122 |
+
if row is None:
|
| 123 |
+
await ws.close(code=status.WS_1008_POLICY_VIOLATION,
|
| 124 |
+
reason="session-not-found")
|
| 125 |
+
return None
|
| 126 |
+
if not resume_token or row.get("resume_token") != resume_token:
|
| 127 |
+
await ws.close(code=status.WS_1008_POLICY_VIOLATION,
|
| 128 |
+
reason="bad-resume-token")
|
| 129 |
+
return None
|
| 130 |
+
if row.get("status") == "ended":
|
| 131 |
+
await ws.close(code=status.WS_1008_POLICY_VIOLATION,
|
| 132 |
+
reason="session-ended")
|
| 133 |
+
return None
|
| 134 |
+
# Resume window: an "interrupted" session is only acceptable if the
|
| 135 |
+
# drop happened within the configured window.
|
| 136 |
+
if row.get("status") == "interrupted":
|
| 137 |
+
last_update = int(row.get("updated_at") or 0)
|
| 138 |
+
if _now_ms() - last_update > cfg.resume_window_sec * 1000:
|
| 139 |
+
service.mark_ended_by_timeout(session_id, "resume_expired")
|
| 140 |
+
await ws.close(code=status.WS_1008_POLICY_VIOLATION,
|
| 141 |
+
reason="resume-expired")
|
| 142 |
+
return None
|
| 143 |
+
return row
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
async def run_session_ws(
|
| 147 |
+
ws: WebSocket,
|
| 148 |
+
session_id: str,
|
| 149 |
+
resume_token: Optional[str],
|
| 150 |
+
cfg: VoiceCallConfig,
|
| 151 |
+
) -> None:
|
| 152 |
+
"""Main WS loop for one session. Called from router after
|
| 153 |
+
``await ws.accept()``.
|
| 154 |
+
|
| 155 |
+
Ownership is established by the resume token alone (token is only
|
| 156 |
+
ever handed to the user who created the session). The per-turn
|
| 157 |
+
chat request also carries the caller's bearer β so the chat
|
| 158 |
+
endpoint does its own auth and refuses foreign personas even if
|
| 159 |
+
the token somehow leaks."""
|
| 160 |
+
session = await _resolve_session_for_ws(ws, session_id, resume_token, cfg)
|
| 161 |
+
if session is None:
|
| 162 |
+
return # close already issued
|
| 163 |
+
|
| 164 |
+
# Extract the bearer from the WS query so turn.py can re-use it.
|
| 165 |
+
# Browsers can't attach custom headers to WS; the client passes
|
| 166 |
+
# ?token=<jwt> alongside ?resume_token=.
|
| 167 |
+
query_bearer = ws.query_params.get("token") or None
|
| 168 |
+
|
| 169 |
+
sid = session["id"]
|
| 170 |
+
_clog("lifecycle_open", sid=sid, persona_id=session.get("persona_id"))
|
| 171 |
+
service.mark_live(sid)
|
| 172 |
+
await _send(ws, "call.state", {"status": "live"}, session_id=sid)
|
| 173 |
+
_clog("lifecycle_ready", sid=sid)
|
| 174 |
+
|
| 175 |
+
# ββ "Theory of answering" hook ββββββββββββββββββββββββββββββββ
|
| 176 |
+
# When persona_call is on, the persona answers first β the way a
|
| 177 |
+
# real callee picks up the phone β instead of sitting silent
|
| 178 |
+
# until the caller speaks. The opening text comes from the
|
| 179 |
+
# curated bank in persona_call/openings.py; selection is
|
| 180 |
+
# deterministic (seeded by session_id) + persona-aware + rotated
|
| 181 |
+
# against the anti-repetition ledger, so the same caller never
|
| 182 |
+
# hears the same greeting twice in a row.
|
| 183 |
+
try:
|
| 184 |
+
from ..persona_call import (
|
| 185 |
+
config as _pc_cfg_mod,
|
| 186 |
+
context as _pc_context,
|
| 187 |
+
facets as _pc_facets,
|
| 188 |
+
openings as _pc_openings,
|
| 189 |
+
store as _pc_store,
|
| 190 |
+
)
|
| 191 |
+
_pc_cfg = _pc_cfg_mod.load()
|
| 192 |
+
if _pc_cfg.enabled:
|
| 193 |
+
_pc_store.ensure_schema()
|
| 194 |
+
_pc_store.ensure_state(sid)
|
| 195 |
+
_state = _pc_store.get_state(sid) or {}
|
| 196 |
+
forbidden = list(_state.get("recent_openers") or [])
|
| 197 |
+
_pc_env = _pc_context.compute_env(tz=None, weeks_since_last_call=-1)
|
| 198 |
+
_pc_facets_obj = _pc_facets.for_persona_id(session.get("persona_id"))
|
| 199 |
+
# We need the template *id* too so we can push it into
|
| 200 |
+
# the ledger, so call choose() (not render() directly).
|
| 201 |
+
tpl = _pc_openings.choose(
|
| 202 |
+
_pc_facets_obj, _pc_env,
|
| 203 |
+
session_id=sid,
|
| 204 |
+
turn_index=1,
|
| 205 |
+
forbidden_ids=forbidden,
|
| 206 |
+
)
|
| 207 |
+
greeting_text = None
|
| 208 |
+
if tpl is not None:
|
| 209 |
+
greeting_text = _pc_openings._interpolate(
|
| 210 |
+
tpl.text,
|
| 211 |
+
# Fall back to "Assistant" only if the session row
|
| 212 |
+
# has no persona_id β matches the frontend label.
|
| 213 |
+
(session.get("persona_id") or "Assistant").replace("persona:", "").strip(),
|
| 214 |
+
)
|
| 215 |
+
# Append the chosen id to the opener ledger, keeping
|
| 216 |
+
# only the last N so the ledger doesn't grow
|
| 217 |
+
# unbounded across long-lived sessions.
|
| 218 |
+
window = max(1, _pc_cfg.opener_ledger_window)
|
| 219 |
+
new_openers = (list(forbidden) + [tpl.id])[-window:]
|
| 220 |
+
_pc_store.update_state(sid, recent_openers=new_openers)
|
| 221 |
+
if greeting_text:
|
| 222 |
+
await _send(ws, "transcript.final",
|
| 223 |
+
{"role": "assistant", "text": greeting_text},
|
| 224 |
+
session_id=sid)
|
| 225 |
+
_clog("lifecycle_greet", sid=sid, text_len=len(greeting_text))
|
| 226 |
+
except Exception as _pc_err:
|
| 227 |
+
# Shadow-fail β the call still works without a greeting.
|
| 228 |
+
logger.warning("[persona_call] opening greeting skipped: %s", _pc_err)
|
| 229 |
+
|
| 230 |
+
# Heartbeat keeps idle TCP paths alive on mobile networks.
|
| 231 |
+
hb_task = asyncio.create_task(_heartbeat_loop(ws, sid, cfg))
|
| 232 |
+
|
| 233 |
+
try:
|
| 234 |
+
while True:
|
| 235 |
+
try:
|
| 236 |
+
text = await asyncio.wait_for(
|
| 237 |
+
ws.receive_text(),
|
| 238 |
+
timeout=cfg.idle_timeout_sec,
|
| 239 |
+
)
|
| 240 |
+
except asyncio.TimeoutError:
|
| 241 |
+
await _send_error(ws, sid, "idle_timeout",
|
| 242 |
+
"No activity for a while β ending call.")
|
| 243 |
+
service.end_session(sid=sid, user_id=session["user_id"],
|
| 244 |
+
reason="idle")
|
| 245 |
+
await _send(ws, "call.state", {"status": "ended",
|
| 246 |
+
"reason": "idle"},
|
| 247 |
+
session_id=sid)
|
| 248 |
+
break
|
| 249 |
+
|
| 250 |
+
try:
|
| 251 |
+
env = json.loads(text)
|
| 252 |
+
except json.JSONDecodeError:
|
| 253 |
+
await _send_error(ws, sid, "bad_json", "Invalid JSON envelope.")
|
| 254 |
+
continue
|
| 255 |
+
|
| 256 |
+
evt_type = env.get("type")
|
| 257 |
+
payload = env.get("payload") or {}
|
| 258 |
+
|
| 259 |
+
if evt_type == "call.control":
|
| 260 |
+
action = (payload.get("action") or "").strip()
|
| 261 |
+
if action == "end":
|
| 262 |
+
service.end_session(sid=sid, user_id=session["user_id"],
|
| 263 |
+
reason="user_ended")
|
| 264 |
+
await _send(ws, "call.state", {"status": "ended",
|
| 265 |
+
"reason": "user_ended"},
|
| 266 |
+
session_id=sid)
|
| 267 |
+
_clog("lifecycle_hangup", sid=sid, reason="user_ended")
|
| 268 |
+
break
|
| 269 |
+
if action == "ping":
|
| 270 |
+
await _send(ws, "pong", {}, session_id=sid)
|
| 271 |
+
|
| 272 |
+
elif evt_type == "ui.state":
|
| 273 |
+
# Pure bookkeeping. Client tells us muted / speaker_on /
|
| 274 |
+
# backgrounded; we persist only the type (privacy).
|
| 275 |
+
pass
|
| 276 |
+
|
| 277 |
+
elif evt_type == "transcript.final":
|
| 278 |
+
text_in = (payload.get("text") or "").strip()
|
| 279 |
+
if not text_in:
|
| 280 |
+
continue
|
| 281 |
+
# Protocol sentinels are NOT user utterances β they
|
| 282 |
+
# exist solely so the frontend can signal state
|
| 283 |
+
# transitions (phase machine nudges, etc.) without
|
| 284 |
+
# the LLM reacting to the literal string. Drop and
|
| 285 |
+
# continue; the phase machine already advances on the
|
| 286 |
+
# first real user utterance, so no special handling
|
| 287 |
+
# is required here beyond not running the turn.
|
| 288 |
+
if text_in in _PROTOCOL_SENTINELS:
|
| 289 |
+
continue
|
| 290 |
+
_clog("turn_user_in", sid=sid, text_len=len(text_in))
|
| 291 |
+
model = (
|
| 292 |
+
payload.get("model")
|
| 293 |
+
or session.get("persona_id")
|
| 294 |
+
and f"persona:{session['persona_id']}"
|
| 295 |
+
or "llama3:8b"
|
| 296 |
+
)
|
| 297 |
+
|
| 298 |
+
# ββ persona_call hook βββββββββββββββββββββββββββββββ
|
| 299 |
+
# Additive, behind its own feature flag. Composes a
|
| 300 |
+
# per-turn system SUFFIX (never touches the persona's
|
| 301 |
+
# base system prompt) and β if cfg.apply is True β
|
| 302 |
+
# appends it as an extra system message on the chat
|
| 303 |
+
# call. Wraps the turn in a filler scheduler so a long
|
| 304 |
+
# LLM wait produces a "hmmβ¦" event within 600 ms.
|
| 305 |
+
pc_additional_system: Optional[str] = None
|
| 306 |
+
pc_persona_id: Optional[str] = session.get("persona_id")
|
| 307 |
+
try:
|
| 308 |
+
from ..persona_call import (
|
| 309 |
+
config as _pc_cfg_mod,
|
| 310 |
+
context as _pc_context,
|
| 311 |
+
directive as _pc_directive,
|
| 312 |
+
facets as _pc_facets,
|
| 313 |
+
latency as _pc_latency,
|
| 314 |
+
)
|
| 315 |
+
_pc_cfg = _pc_cfg_mod.load()
|
| 316 |
+
except Exception:
|
| 317 |
+
_pc_cfg = None # persona_call not importable; skip
|
| 318 |
+
|
| 319 |
+
_pc_facets_obj = None
|
| 320 |
+
if _pc_cfg is not None and _pc_cfg.enabled:
|
| 321 |
+
try:
|
| 322 |
+
_pc_env = _pc_context.compute_env(
|
| 323 |
+
tz=(session.get("client_platform") and None),
|
| 324 |
+
weeks_since_last_call=-1,
|
| 325 |
+
)
|
| 326 |
+
_pc_facets_obj = _pc_facets.for_persona_id(pc_persona_id)
|
| 327 |
+
composed = _pc_directive.compose(
|
| 328 |
+
session_id=sid,
|
| 329 |
+
persona_id=pc_persona_id,
|
| 330 |
+
user_text=text_in,
|
| 331 |
+
env=_pc_env,
|
| 332 |
+
cfg=_pc_cfg,
|
| 333 |
+
facets=_pc_facets_obj,
|
| 334 |
+
)
|
| 335 |
+
if _pc_cfg.apply and composed.system_suffix:
|
| 336 |
+
pc_additional_system = composed.system_suffix
|
| 337 |
+
except Exception as _pc_err:
|
| 338 |
+
# Shadow fail β turn still runs without the
|
| 339 |
+
# phone-call suffix. persona_call must never
|
| 340 |
+
# break an otherwise-working call.
|
| 341 |
+
logger.warning(
|
| 342 |
+
"[persona_call] compose skipped: %s", _pc_err
|
| 343 |
+
)
|
| 344 |
+
|
| 345 |
+
async def _pc_send(env_: dict) -> None:
|
| 346 |
+
try:
|
| 347 |
+
await ws.send_text(
|
| 348 |
+
json.dumps(env_, separators=(",", ":"))
|
| 349 |
+
)
|
| 350 |
+
except Exception:
|
| 351 |
+
pass
|
| 352 |
+
|
| 353 |
+
# ββ Streaming vs unary turn β additive branch ββββββ
|
| 354 |
+
# When VOICE_CALL_STREAMING_ENABLED=true the turn runs
|
| 355 |
+
# through turn_stream.run_turn_streaming and emits
|
| 356 |
+
# ``assistant.partial`` + ``assistant.turn_end`` envelopes
|
| 357 |
+
# so the client's streaming TTS can begin speaking long
|
| 358 |
+
# before the full reply has arrived. When the flag is
|
| 359 |
+
# off we take the original path byte-for-byte.
|
| 360 |
+
if cfg.streaming_enabled:
|
| 361 |
+
turn_id = f"t_{uuid.uuid4().hex[:12]}"
|
| 362 |
+
token = _barge_in.new_token(sid, turn_id)
|
| 363 |
+
assistant_text_parts: list[str] = []
|
| 364 |
+
end_reason = "complete"
|
| 365 |
+
try:
|
| 366 |
+
if (
|
| 367 |
+
_pc_cfg is not None
|
| 368 |
+
and _pc_cfg.enabled
|
| 369 |
+
and _pc_facets_obj is not None
|
| 370 |
+
):
|
| 371 |
+
# Filler scheduler still wraps streaming β
|
| 372 |
+
# if the FIRST delta takes > N ms the
|
| 373 |
+
# client still gets a ``hmmβ¦`` event.
|
| 374 |
+
async with _pc_latency.FillerScheduler(
|
| 375 |
+
send=_pc_send,
|
| 376 |
+
facets=_pc_facets_obj,
|
| 377 |
+
cfg=_pc_cfg,
|
| 378 |
+
session_id=sid,
|
| 379 |
+
):
|
| 380 |
+
index = 0
|
| 381 |
+
async for delta in turn_stream.run_turn_streaming(
|
| 382 |
+
user_text=text_in,
|
| 383 |
+
model=model,
|
| 384 |
+
auth_bearer=query_bearer,
|
| 385 |
+
additional_system=pc_additional_system,
|
| 386 |
+
cancel_token=token,
|
| 387 |
+
):
|
| 388 |
+
if token.is_cancelled():
|
| 389 |
+
break
|
| 390 |
+
assistant_text_parts.append(delta)
|
| 391 |
+
await _send(
|
| 392 |
+
ws, "assistant.partial",
|
| 393 |
+
{
|
| 394 |
+
"turn_id": turn_id,
|
| 395 |
+
"delta": delta,
|
| 396 |
+
"index": index,
|
| 397 |
+
},
|
| 398 |
+
session_id=sid,
|
| 399 |
+
)
|
| 400 |
+
index += 1
|
| 401 |
+
else:
|
| 402 |
+
index = 0
|
| 403 |
+
async for delta in turn_stream.run_turn_streaming(
|
| 404 |
+
user_text=text_in,
|
| 405 |
+
model=model,
|
| 406 |
+
auth_bearer=query_bearer,
|
| 407 |
+
additional_system=pc_additional_system,
|
| 408 |
+
cancel_token=token,
|
| 409 |
+
):
|
| 410 |
+
if token.is_cancelled():
|
| 411 |
+
break
|
| 412 |
+
assistant_text_parts.append(delta)
|
| 413 |
+
await _send(
|
| 414 |
+
ws, "assistant.partial",
|
| 415 |
+
{
|
| 416 |
+
"turn_id": turn_id,
|
| 417 |
+
"delta": delta,
|
| 418 |
+
"index": index,
|
| 419 |
+
},
|
| 420 |
+
session_id=sid,
|
| 421 |
+
)
|
| 422 |
+
index += 1
|
| 423 |
+
if token.is_cancelled():
|
| 424 |
+
end_reason = "cancelled"
|
| 425 |
+
except Exception as exc:
|
| 426 |
+
logger.warning("[voice_call] stream turn failed: %s", exc)
|
| 427 |
+
end_reason = "error"
|
| 428 |
+
finally:
|
| 429 |
+
# Always emit turn_end so the client can close
|
| 430 |
+
# the turn bookkeeping cleanly.
|
| 431 |
+
_barge_in.clear_session(sid)
|
| 432 |
+
assistant_text = "".join(assistant_text_parts)
|
| 433 |
+
await _send(
|
| 434 |
+
ws, "assistant.turn_end",
|
| 435 |
+
{
|
| 436 |
+
"turn_id": turn_id,
|
| 437 |
+
"reason": end_reason,
|
| 438 |
+
"full_text": assistant_text,
|
| 439 |
+
},
|
| 440 |
+
session_id=sid,
|
| 441 |
+
)
|
| 442 |
+
_clog(
|
| 443 |
+
"turn_assistant_out",
|
| 444 |
+
sid=sid,
|
| 445 |
+
turn_id=turn_id,
|
| 446 |
+
reason=end_reason,
|
| 447 |
+
text_len=len(assistant_text),
|
| 448 |
+
route="stream",
|
| 449 |
+
)
|
| 450 |
+
else:
|
| 451 |
+
try:
|
| 452 |
+
if (
|
| 453 |
+
_pc_cfg is not None
|
| 454 |
+
and _pc_cfg.enabled
|
| 455 |
+
and _pc_facets_obj is not None
|
| 456 |
+
):
|
| 457 |
+
async with _pc_latency.FillerScheduler(
|
| 458 |
+
send=_pc_send,
|
| 459 |
+
facets=_pc_facets_obj,
|
| 460 |
+
cfg=_pc_cfg,
|
| 461 |
+
session_id=sid,
|
| 462 |
+
):
|
| 463 |
+
assistant_text = await turn.run_turn(
|
| 464 |
+
user_text=text_in,
|
| 465 |
+
model=model,
|
| 466 |
+
auth_bearer=query_bearer,
|
| 467 |
+
additional_system=pc_additional_system,
|
| 468 |
+
)
|
| 469 |
+
else:
|
| 470 |
+
assistant_text = await turn.run_turn(
|
| 471 |
+
user_text=text_in,
|
| 472 |
+
model=model,
|
| 473 |
+
auth_bearer=query_bearer,
|
| 474 |
+
additional_system=pc_additional_system,
|
| 475 |
+
)
|
| 476 |
+
except Exception as exc:
|
| 477 |
+
logger.warning("[voice_call] turn failed: %s", exc)
|
| 478 |
+
await _send_error(ws, sid, "turn_failed", str(exc)[:400])
|
| 479 |
+
continue
|
| 480 |
+
|
| 481 |
+
await _send(ws, "transcript.final",
|
| 482 |
+
{"role": "assistant", "text": assistant_text},
|
| 483 |
+
session_id=sid)
|
| 484 |
+
_clog(
|
| 485 |
+
"turn_assistant_out",
|
| 486 |
+
sid=sid,
|
| 487 |
+
text_len=len(assistant_text),
|
| 488 |
+
route="unary",
|
| 489 |
+
)
|
| 490 |
+
|
| 491 |
+
# Record the reply into the anti-repetition ledger so
|
| 492 |
+
# the NEXT turn can forbid the just-used opener/ack.
|
| 493 |
+
if _pc_cfg is not None and _pc_cfg.enabled:
|
| 494 |
+
try:
|
| 495 |
+
_pc_directive.record_persona_reply(
|
| 496 |
+
session_id=sid,
|
| 497 |
+
persona_id=pc_persona_id,
|
| 498 |
+
reply=assistant_text,
|
| 499 |
+
cfg=_pc_cfg,
|
| 500 |
+
)
|
| 501 |
+
except Exception as _pc_err:
|
| 502 |
+
logger.warning(
|
| 503 |
+
"[persona_call] record_reply skipped: %s",
|
| 504 |
+
_pc_err,
|
| 505 |
+
)
|
| 506 |
+
|
| 507 |
+
elif evt_type == "user.barge_in":
|
| 508 |
+
# Phase 3 β cooperative cancel. Client's VAD detected
|
| 509 |
+
# user speech-start while the persona was mid-reply.
|
| 510 |
+
# Β§ 4.4 of the streaming design doc.
|
| 511 |
+
if not (cfg.streaming_enabled and cfg.barge_in_enabled):
|
| 512 |
+
continue
|
| 513 |
+
turn_id = (payload.get("turn_id") or "").strip()
|
| 514 |
+
if not turn_id:
|
| 515 |
+
continue
|
| 516 |
+
cancelled = _barge_in.cancel_active(sid, turn_id)
|
| 517 |
+
if cancelled:
|
| 518 |
+
# Emit the ack IMMEDIATELY β client stops TTS on
|
| 519 |
+
# receipt, shaving ~50-100 ms off the perceived
|
| 520 |
+
# barge-in latency vs waiting for the generator
|
| 521 |
+
# to notice the event on its next poll.
|
| 522 |
+
await _send(
|
| 523 |
+
ws, "assistant.cancel",
|
| 524 |
+
{"turn_id": turn_id, "cause": "user_barge_in"},
|
| 525 |
+
session_id=sid,
|
| 526 |
+
)
|
| 527 |
+
_clog("barge_in", sid=sid, turn_id=turn_id, cause="user_barge_in")
|
| 528 |
+
# Else: stale turn_id (turn already ended). Silent drop.
|
| 529 |
+
|
| 530 |
+
elif evt_type == "transcript.partial":
|
| 531 |
+
# Secondary barge-in signal β if the client's explicit
|
| 532 |
+
# user.barge_in didn't arrive but the STT started
|
| 533 |
+
# decoding words, that also means the user spoke.
|
| 534 |
+
# Only fires while a turn is active.
|
| 535 |
+
if not (cfg.streaming_enabled and cfg.barge_in_enabled):
|
| 536 |
+
continue
|
| 537 |
+
active = _barge_in.get_active(sid)
|
| 538 |
+
if active is not None and not active.is_cancelled():
|
| 539 |
+
cancelled = _barge_in.cancel_active(sid, active.turn_id)
|
| 540 |
+
if cancelled:
|
| 541 |
+
await _send(
|
| 542 |
+
ws, "assistant.cancel",
|
| 543 |
+
{
|
| 544 |
+
"turn_id": active.turn_id,
|
| 545 |
+
"cause": "user_partial",
|
| 546 |
+
},
|
| 547 |
+
session_id=sid,
|
| 548 |
+
)
|
| 549 |
+
|
| 550 |
+
else:
|
| 551 |
+
# Forward-compat: unknown event type is a no-op, not an
|
| 552 |
+
# error. Older servers + newer clients stay friendly.
|
| 553 |
+
pass
|
| 554 |
+
except WebSocketDisconnect:
|
| 555 |
+
# Client dropped β park the session as 'interrupted' so a resume
|
| 556 |
+
# request within the window can pick it back up.
|
| 557 |
+
_barge_in.clear_session(sid)
|
| 558 |
+
service.mark_interrupted(sid)
|
| 559 |
+
except Exception as exc:
|
| 560 |
+
logger.exception("[voice_call] ws error for %s: %s", sid, exc)
|
| 561 |
+
try:
|
| 562 |
+
await _send_error(ws, sid, "internal", "Internal error")
|
| 563 |
+
except Exception:
|
| 564 |
+
pass
|
| 565 |
+
_barge_in.clear_session(sid)
|
| 566 |
+
service.mark_interrupted(sid)
|
| 567 |
+
finally:
|
| 568 |
+
_clog("lifecycle_close", sid=sid)
|
| 569 |
+
_barge_in.clear_session(sid)
|
| 570 |
+
hb_task.cancel()
|
| 571 |
+
try:
|
| 572 |
+
await hb_task
|
| 573 |
+
except Exception:
|
| 574 |
+
pass
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
async def _heartbeat_loop(ws: WebSocket, session_id: str, cfg: VoiceCallConfig) -> None:
|
| 578 |
+
"""Send a server-initiated ping every N seconds. Doubles as a
|
| 579 |
+
liveness check + hard-duration-cap enforcement."""
|
| 580 |
+
while True:
|
| 581 |
+
await asyncio.sleep(HEARTBEAT_INTERVAL_SEC)
|
| 582 |
+
row = store._select_session(session_id)
|
| 583 |
+
if row is None or row["status"] == "ended":
|
| 584 |
+
return
|
| 585 |
+
# Enforce hard cap.
|
| 586 |
+
started = int(row.get("started_at") or 0)
|
| 587 |
+
if started and (_now_ms() - started) >= cfg.max_duration_sec * 1000:
|
| 588 |
+
try:
|
| 589 |
+
await _send(ws, "call.state",
|
| 590 |
+
{"status": "ending", "reason": "max_duration"},
|
| 591 |
+
session_id=session_id)
|
| 592 |
+
except Exception:
|
| 593 |
+
return
|
| 594 |
+
service.end_session(sid=session_id, user_id=row["user_id"],
|
| 595 |
+
reason="max_duration")
|
| 596 |
+
try:
|
| 597 |
+
await ws.close(code=status.WS_1000_NORMAL_CLOSURE,
|
| 598 |
+
reason="max-duration")
|
| 599 |
+
except Exception:
|
| 600 |
+
pass
|
| 601 |
+
return
|
| 602 |
+
try:
|
| 603 |
+
await _send(ws, "ping", {}, session_id=session_id)
|
| 604 |
+
except Exception:
|
| 605 |
+
return
|
frontend/.cache-bust
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
|
|
|
|
| 1 |
+
1776516992
|
frontend/src/ui/App.tsx
CHANGED
|
@@ -21,6 +21,7 @@ import {
|
|
| 21 |
Copy,
|
| 22 |
RotateCw,
|
| 23 |
PenLine,
|
|
|
|
| 24 |
Users,
|
| 25 |
EyeOff,
|
| 26 |
BookOpen,
|
|
@@ -34,6 +35,9 @@ import UserAvatar from './components/UserAvatar'
|
|
| 34 |
import AccountMenu, { type AccountMenuUser } from './components/AccountMenu'
|
| 35 |
import { useAuth } from './components/AuthGate'
|
| 36 |
import VoiceMode, { stripMarkdownForSpeech } from './VoiceModeGrok'
|
|
|
|
|
|
|
|
|
|
| 37 |
// Legacy voice mode available as: import VoiceModeLegacy from './VoiceModeLegacy'
|
| 38 |
import ProjectsView from './ProjectsView'
|
| 39 |
import ImagineView from './Imagine'
|
|
@@ -113,6 +117,22 @@ export type Msg = {
|
|
| 113 |
intent: 'generate_images' | 'generate_videos'
|
| 114 |
prompt: string
|
| 115 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
}
|
| 117 |
|
| 118 |
// ---------------------------------------------------------------------------
|
|
@@ -1574,12 +1594,92 @@ function useCopyMessage(timeoutMs = 900) {
|
|
| 1574 |
return { copied, copy }
|
| 1575 |
}
|
| 1576 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1577 |
function ChatState({
|
| 1578 |
messages,
|
| 1579 |
setLightbox,
|
| 1580 |
endRef,
|
| 1581 |
mode,
|
| 1582 |
onNewConversation,
|
|
|
|
| 1583 |
onRetryMessage,
|
| 1584 |
chatSettings,
|
| 1585 |
onUpdateChatSettings,
|
|
@@ -1598,6 +1698,9 @@ function ChatState({
|
|
| 1598 |
endRef: React.RefObject<HTMLDivElement>
|
| 1599 |
mode: Mode
|
| 1600 |
onNewConversation: () => void
|
|
|
|
|
|
|
|
|
|
| 1601 |
onRetryMessage: (id: string) => void
|
| 1602 |
chatSettings: ChatScopedSettings
|
| 1603 |
onUpdateChatSettings: (next: ChatScopedSettings) => void
|
|
@@ -1613,6 +1716,9 @@ function ChatState({
|
|
| 1613 |
}) {
|
| 1614 |
const { copied, copy } = useCopyMessage()
|
| 1615 |
const [chatSettingsOpen, setChatSettingsOpen] = useState(false)
|
|
|
|
|
|
|
|
|
|
| 1616 |
// Local overrides for view-pack angle switching (keyed by message id)
|
| 1617 |
const [angleOverrides, setAngleOverrides] = useState<Record<string, { angle: string; url: string }>>({})
|
| 1618 |
|
|
@@ -1650,9 +1756,24 @@ function ChatState({
|
|
| 1650 |
|
| 1651 |
return (
|
| 1652 |
<div className="flex flex-col h-full w-full max-w-[52rem] mx-auto">
|
| 1653 |
-
{/* Top-right fixed:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1654 |
<div className="fixed top-3 right-5 z-50">
|
| 1655 |
-
<div className="relative flex items-center gap-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1656 |
<button
|
| 1657 |
type="button"
|
| 1658 |
onClick={() => setChatSettingsOpen((v) => !v)}
|
|
@@ -1701,6 +1822,20 @@ function ChatState({
|
|
| 1701 |
key={m.id}
|
| 1702 |
className={`flex gap-5 ${m.role === 'user' ? 'justify-end' : 'justify-start'}`}
|
| 1703 |
>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1704 |
{m.role === 'assistant' ? (
|
| 1705 |
<div className="w-8 h-8 rounded-full bg-white text-black flex items-center justify-center flex-shrink-0 font-bold text-sm mt-1">
|
| 1706 |
/
|
|
@@ -1858,6 +1993,7 @@ function ChatState({
|
|
| 1858 |
) : null}
|
| 1859 |
</div>
|
| 1860 |
)}
|
|
|
|
| 1861 |
</div>
|
| 1862 |
))}
|
| 1863 |
<div ref={endRef} className="h-6" />
|
|
@@ -2040,6 +2176,22 @@ export default function App() {
|
|
| 2040 |
})
|
| 2041 |
const [voiceConversationId, setVoiceConversationId] = useState<string>(() => uuid())
|
| 2042 |
const [lightbox, setLightbox] = useState<string | null>(null)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2043 |
|
| 2044 |
// Persona Integration β "Save as Persona Avatar" from AvatarStudio (additive)
|
| 2045 |
const [saveAsPersonaData, setSaveAsPersonaData] = useState<{ item: GalleryItem; outfits: GalleryItem[]; batchSiblings?: GalleryItem[] } | null>(null)
|
|
@@ -2311,6 +2463,26 @@ export default function App() {
|
|
| 2311 |
// Track last spoken message to avoid re-speaking
|
| 2312 |
const lastSpokenMessageIdRef = useRef<string | null>(null)
|
| 2313 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2314 |
// When a voice session is explicitly chosen from Session Hub, skip the auto-link resolve
|
| 2315 |
// to prevent the useEffect from overwriting the chosen session with the active one.
|
| 2316 |
const voiceSessionExplicitRef = useRef(false)
|
|
@@ -3985,10 +4157,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 3985 |
[authHeaders, conversationId, messages, settings.backendUrl, settings.funMode, settingsDraft]
|
| 3986 |
)
|
| 3987 |
|
| 3988 |
-
// TTS for assistant responses (speak-once pattern)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3989 |
useEffect(() => {
|
| 3990 |
-
|
| 3991 |
-
if (mode !== 'voice') return
|
| 3992 |
|
| 3993 |
const ttsEnabled = settingsDraft.ttsEnabled ?? true
|
| 3994 |
if (!ttsEnabled || !window.SpeechService) return
|
|
@@ -4001,6 +4184,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 4001 |
lastMessage.role === 'assistant' &&
|
| 4002 |
!lastMessage.pending &&
|
| 4003 |
lastMessage.text &&
|
|
|
|
| 4004 |
lastMessage.id !== lastSpokenMessageIdRef.current
|
| 4005 |
) {
|
| 4006 |
// Mark this message as spoken
|
|
@@ -4009,9 +4193,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 4009 |
// Speak the assistant's response β strip markdown images/links so TTS
|
| 4010 |
// doesn't read raw URLs aloud. Images are shown visually instead.
|
| 4011 |
const speechText = stripMarkdownForSpeech(lastMessage.text)
|
| 4012 |
-
if (speechText)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4013 |
}
|
| 4014 |
-
}, [messages, settingsDraft.ttsEnabled, mode])
|
| 4015 |
|
| 4016 |
const uploadAndSend = useCallback(
|
| 4017 |
async (file: File) => {
|
|
@@ -4489,7 +4685,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 4489 |
|
| 4490 |
{/* Top-right Project Indicator - Only shown in chat mode when a project is active */}
|
| 4491 |
{mode === 'chat' && (
|
| 4492 |
-
<header className="absolute top-0 left-0 right-0 pr-[
|
| 4493 |
{/* Project Indicator */}
|
| 4494 |
{(() => {
|
| 4495 |
const currentProjectId = localStorage.getItem('homepilot_current_project')
|
|
@@ -5049,6 +5245,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 5049 |
endRef={endRef}
|
| 5050 |
mode={mode}
|
| 5051 |
onNewConversation={onNewConversation}
|
|
|
|
| 5052 |
onRetryMessage={retryFailedMessage}
|
| 5053 |
chatSettings={chatSettings}
|
| 5054 |
onUpdateChatSettings={updateChatSettings}
|
|
@@ -5124,6 +5321,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 5124 |
endRef={endRef}
|
| 5125 |
mode={mode}
|
| 5126 |
onNewConversation={onNewConversation}
|
|
|
|
| 5127 |
onRetryMessage={retryFailedMessage}
|
| 5128 |
chatSettings={chatSettings}
|
| 5129 |
onUpdateChatSettings={updateChatSettings}
|
|
@@ -5149,6 +5347,91 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
|
|
| 5149 |
/>
|
| 5150 |
) : null}
|
| 5151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5152 |
{/* Save as Persona Avatar modal (from AvatarStudio gallery) */}
|
| 5153 |
{saveAsPersonaData && (
|
| 5154 |
<SaveAsPersonaModal
|
|
|
|
| 21 |
Copy,
|
| 22 |
RotateCw,
|
| 23 |
PenLine,
|
| 24 |
+
Phone,
|
| 25 |
Users,
|
| 26 |
EyeOff,
|
| 27 |
BookOpen,
|
|
|
|
| 35 |
import AccountMenu, { type AccountMenuUser } from './components/AccountMenu'
|
| 36 |
import { useAuth } from './components/AuthGate'
|
| 37 |
import VoiceMode, { stripMarkdownForSpeech } from './VoiceModeGrok'
|
| 38 |
+
import CallOverlay from './CallOverlay'
|
| 39 |
+
import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log'
|
| 40 |
+
import PostCallCard from './phone/PostCallCard'
|
| 41 |
// Legacy voice mode available as: import VoiceModeLegacy from './VoiceModeLegacy'
|
| 42 |
import ProjectsView from './ProjectsView'
|
| 43 |
import ImagineView from './Imagine'
|
|
|
|
| 117 |
intent: 'generate_images' | 'generate_videos'
|
| 118 |
prompt: string
|
| 119 |
}
|
| 120 |
+
// "Call memory card" β when present the message renders as an
|
| 121 |
+
// inline record of a just-ended voice call instead of a regular
|
| 122 |
+
// bubble. Additive: untouched by every existing code path.
|
| 123 |
+
callMemory?: {
|
| 124 |
+
durationSec: number
|
| 125 |
+
/** Epoch-ms timestamp of when the call ended. Rendered as an
|
| 126 |
+
* auditable stamp (e.g. "7:36 PM") in the card header. */
|
| 127 |
+
endedAt?: number
|
| 128 |
+
/** Persona the user was speaking with during this call β feeds
|
| 129 |
+
* PostCallCard's "Call with X" / "X remembers" headers. */
|
| 130 |
+
personaName?: string
|
| 131 |
+
/** Optional transcript the user can expand inline from the card.
|
| 132 |
+
* Populated by App.tsx at call-end by slicing chatMessages
|
| 133 |
+
* between the call's open + close timestamps. */
|
| 134 |
+
transcript?: Array<{ who: 'user' | 'assistant'; text: string }>
|
| 135 |
+
}
|
| 136 |
}
|
| 137 |
|
| 138 |
// ---------------------------------------------------------------------------
|
|
|
|
| 1594 |
return { copied, copy }
|
| 1595 |
}
|
| 1596 |
|
| 1597 |
+
/**
|
| 1598 |
+
* CallMemoryCard β enterprise-style inline record of a just-ended
|
| 1599 |
+
* call. Rendered inside the chat stream as a neutral system event
|
| 1600 |
+
* rather than a floating toast or an emotional "moment" card. The
|
| 1601 |
+
* tone is auditable, not expressive: the card states what happened,
|
| 1602 |
+
* when, and offers one standard system action to resume.
|
| 1603 |
+
*/
|
| 1604 |
+
function CallMemoryCard({
|
| 1605 |
+
durationSec,
|
| 1606 |
+
endedAt,
|
| 1607 |
+
onSpeakAgain,
|
| 1608 |
+
}: {
|
| 1609 |
+
durationSec: number
|
| 1610 |
+
endedAt?: number
|
| 1611 |
+
onSpeakAgain?: () => void
|
| 1612 |
+
}) {
|
| 1613 |
+
const fmt = (s: number) => {
|
| 1614 |
+
if (s < 60) return `${s}s`
|
| 1615 |
+
const m = Math.floor(s / 60)
|
| 1616 |
+
const r = s % 60
|
| 1617 |
+
return r === 0 ? `${m}m` : `${m}m ${r}s`
|
| 1618 |
+
}
|
| 1619 |
+
const stamp = endedAt
|
| 1620 |
+
? new Date(endedAt).toLocaleTimeString(undefined, {
|
| 1621 |
+
hour: 'numeric', minute: '2-digit',
|
| 1622 |
+
})
|
| 1623 |
+
: ''
|
| 1624 |
+
return (
|
| 1625 |
+
<div
|
| 1626 |
+
className="flex items-start gap-5 w-full hp-fade-in"
|
| 1627 |
+
role="note"
|
| 1628 |
+
aria-label="Call record"
|
| 1629 |
+
>
|
| 1630 |
+
<div className="w-8 h-8 rounded-full bg-white/5 border border-white/10 text-white/50 flex items-center justify-center flex-shrink-0 mt-1">
|
| 1631 |
+
<Phone size={12} />
|
| 1632 |
+
</div>
|
| 1633 |
+
<div
|
| 1634 |
+
className="max-w-[85%] rounded-[12px] px-4 py-3 bg-white/[0.03] border border-white/10"
|
| 1635 |
+
>
|
| 1636 |
+
{/* Meta row β factual header, low emphasis */}
|
| 1637 |
+
<div className="flex items-center gap-2 text-[11px] text-white/50">
|
| 1638 |
+
<span>Call</span>
|
| 1639 |
+
<span className="text-white/25">Β·</span>
|
| 1640 |
+
<span
|
| 1641 |
+
className="font-mono tabular-nums text-white/70"
|
| 1642 |
+
style={{ fontFeatureSettings: '"tnum"' }}
|
| 1643 |
+
>
|
| 1644 |
+
{fmt(durationSec)}
|
| 1645 |
+
</span>
|
| 1646 |
+
{stamp ? (
|
| 1647 |
+
<>
|
| 1648 |
+
<span className="text-white/25">Β·</span>
|
| 1649 |
+
<span className="font-mono tabular-nums text-white/50">{stamp}</span>
|
| 1650 |
+
</>
|
| 1651 |
+
) : null}
|
| 1652 |
+
</div>
|
| 1653 |
+
{/* Body β neutral factual line, no emotional copy */}
|
| 1654 |
+
<div className="mt-1.5 text-[13.5px] leading-snug text-white/75">
|
| 1655 |
+
Voice session completed
|
| 1656 |
+
</div>
|
| 1657 |
+
{/* CTA β system-style outline button, same weight as header icons */}
|
| 1658 |
+
<div className="mt-3">
|
| 1659 |
+
<button
|
| 1660 |
+
type="button"
|
| 1661 |
+
onClick={onSpeakAgain}
|
| 1662 |
+
disabled={!onSpeakAgain}
|
| 1663 |
+
className="inline-flex items-center gap-1.5 rounded-md bg-transparent hover:bg-white/5 border border-white/15 hover:border-white/25 text-white/80 hover:text-white px-3 py-1 text-[12px] font-medium transition-colors disabled:opacity-40 disabled:cursor-not-allowed"
|
| 1664 |
+
aria-label="Resume call"
|
| 1665 |
+
>
|
| 1666 |
+
<Phone size={12} />
|
| 1667 |
+
Resume call
|
| 1668 |
+
</button>
|
| 1669 |
+
</div>
|
| 1670 |
+
</div>
|
| 1671 |
+
</div>
|
| 1672 |
+
)
|
| 1673 |
+
}
|
| 1674 |
+
|
| 1675 |
+
|
| 1676 |
function ChatState({
|
| 1677 |
messages,
|
| 1678 |
setLightbox,
|
| 1679 |
endRef,
|
| 1680 |
mode,
|
| 1681 |
onNewConversation,
|
| 1682 |
+
onStartCall,
|
| 1683 |
onRetryMessage,
|
| 1684 |
chatSettings,
|
| 1685 |
onUpdateChatSettings,
|
|
|
|
| 1698 |
endRef: React.RefObject<HTMLDivElement>
|
| 1699 |
mode: Mode
|
| 1700 |
onNewConversation: () => void
|
| 1701 |
+
/** Primary call action β opens the live-voice mode. Optional so
|
| 1702 |
+
* callers that don't want the button can omit it. */
|
| 1703 |
+
onStartCall?: () => void
|
| 1704 |
onRetryMessage: (id: string) => void
|
| 1705 |
chatSettings: ChatScopedSettings
|
| 1706 |
onUpdateChatSettings: (next: ChatScopedSettings) => void
|
|
|
|
| 1716 |
}) {
|
| 1717 |
const { copied, copy } = useCopyMessage()
|
| 1718 |
const [chatSettingsOpen, setChatSettingsOpen] = useState(false)
|
| 1719 |
+
const handleStartCall = useCallback(() => {
|
| 1720 |
+
onStartCall?.()
|
| 1721 |
+
}, [onStartCall])
|
| 1722 |
// Local overrides for view-pack angle switching (keyed by message id)
|
| 1723 |
const [angleOverrides, setAngleOverrides] = useState<Record<string, { angle: string; url: string }>>({})
|
| 1724 |
|
|
|
|
| 1756 |
|
| 1757 |
return (
|
| 1758 |
<div className="flex flex-col h-full w-full max-w-[52rem] mx-auto">
|
| 1759 |
+
{/* Top-right fixed: a single flat group of neutral header icons.
|
| 1760 |
+
The call button is intentionally *not* a CTA β it sits inline
|
| 1761 |
+
with the other utility icons and only opens a dedicated call
|
| 1762 |
+
overlay (see CallOverlay). Call mode is a distinct session,
|
| 1763 |
+
not a styling of the header. */}
|
| 1764 |
<div className="fixed top-3 right-5 z-50">
|
| 1765 |
+
<div className="relative flex items-center gap-2">
|
| 1766 |
+
{onStartCall && (
|
| 1767 |
+
<button
|
| 1768 |
+
type="button"
|
| 1769 |
+
onClick={handleStartCall}
|
| 1770 |
+
className="w-9 h-9 flex items-center justify-center rounded-full bg-white/5 border border-white/10 text-white/60 hover:bg-white/10 hover:text-white transition-colors"
|
| 1771 |
+
title="Start call"
|
| 1772 |
+
aria-label="Start call"
|
| 1773 |
+
>
|
| 1774 |
+
<Phone size={16} />
|
| 1775 |
+
</button>
|
| 1776 |
+
)}
|
| 1777 |
<button
|
| 1778 |
type="button"
|
| 1779 |
onClick={() => setChatSettingsOpen((v) => !v)}
|
|
|
|
| 1822 |
key={m.id}
|
| 1823 |
className={`flex gap-5 ${m.role === 'user' ? 'justify-end' : 'justify-start'}`}
|
| 1824 |
>
|
| 1825 |
+
{/* Inline "call memory card" β renders instead of a bubble
|
| 1826 |
+
when this message represents a just-ended voice call.
|
| 1827 |
+
Treated as assistant-aligned (left) so it reads as "a
|
| 1828 |
+
moment we shared" rather than a system toast. */}
|
| 1829 |
+
{m.callMemory ? (
|
| 1830 |
+
<PostCallCard
|
| 1831 |
+
durationSec={m.callMemory.durationSec}
|
| 1832 |
+
endedAt={m.callMemory.endedAt}
|
| 1833 |
+
personaName={m.callMemory.personaName || 'Assistant'}
|
| 1834 |
+
variant="expand"
|
| 1835 |
+
transcript={m.callMemory.transcript}
|
| 1836 |
+
onResume={onStartCall}
|
| 1837 |
+
/>
|
| 1838 |
+
) : (<>
|
| 1839 |
{m.role === 'assistant' ? (
|
| 1840 |
<div className="w-8 h-8 rounded-full bg-white text-black flex items-center justify-center flex-shrink-0 font-bold text-sm mt-1">
|
| 1841 |
/
|
|
|
|
| 1993 |
) : null}
|
| 1994 |
</div>
|
| 1995 |
)}
|
| 1996 |
+
</>)}
|
| 1997 |
</div>
|
| 1998 |
))}
|
| 1999 |
<div ref={endRef} className="h-6" />
|
|
|
|
| 2176 |
})
|
| 2177 |
const [voiceConversationId, setVoiceConversationId] = useState<string>(() => uuid())
|
| 2178 |
const [lightbox, setLightbox] = useState<string | null>(null)
|
| 2179 |
+
// Call mode β a dedicated session distinct from Voice mode. The
|
| 2180 |
+
// header π opens a centered CallOverlay; it does not swap modes.
|
| 2181 |
+
const [callOpen, setCallOpen] = useState(false)
|
| 2182 |
+
// `skipDialing` is flipped on for the "Speak again" re-entry so the
|
| 2183 |
+
// user doesn't have to sit through "callingβ¦" twice in a row.
|
| 2184 |
+
const [callSkipDialing, setCallSkipDialing] = useState(false)
|
| 2185 |
+
// Snapshot of chatMessages.length at the moment a call opens β
|
| 2186 |
+
// used at call-end to slice the transcript of turns that happened
|
| 2187 |
+
// during the call, for PostCallCard's expandable transcript.
|
| 2188 |
+
const callOpenMsgIndexRef = useRef<number>(-1)
|
| 2189 |
+
useEffect(() => {
|
| 2190 |
+
if (callOpen) {
|
| 2191 |
+
callOpenMsgIndexRef.current = chatMessages.length
|
| 2192 |
+
}
|
| 2193 |
+
}, [callOpen]) // intentionally miss chatMessages β we want the
|
| 2194 |
+
// snapshot at OPEN time, not every keystroke after
|
| 2195 |
|
| 2196 |
// Persona Integration β "Save as Persona Avatar" from AvatarStudio (additive)
|
| 2197 |
const [saveAsPersonaData, setSaveAsPersonaData] = useState<{ item: GalleryItem; outfits: GalleryItem[]; batchSiblings?: GalleryItem[] } | null>(null)
|
|
|
|
| 2463 |
// Track last spoken message to avoid re-speaking
|
| 2464 |
const lastSpokenMessageIdRef = useRef<string | null>(null)
|
| 2465 |
|
| 2466 |
+
// Call-open history guard β when the overlay opens, seed the
|
| 2467 |
+
// last-spoken pointer to the CURRENT tail of ``messages``. Without
|
| 2468 |
+
// this the TTS-on-assistant-reply effect treats the previous
|
| 2469 |
+
// chat-history tail as a fresh reply the moment ``callOpen`` flips
|
| 2470 |
+
// true, producing a pre-existing message read-aloud that collides
|
| 2471 |
+
// with CallOverlay's own fallback opener (the "two openings
|
| 2472 |
+
// overlapping" symptom). Only NEW assistant messages that land
|
| 2473 |
+
// DURING the call will be spoken by App.tsx from here on; the
|
| 2474 |
+
// call-itself-opener is owned by CallOverlay via speakOwned().
|
| 2475 |
+
useEffect(() => {
|
| 2476 |
+
if (!callOpen) return
|
| 2477 |
+
const tail = messages[messages.length - 1]
|
| 2478 |
+
lastSpokenMessageIdRef.current = tail ? tail.id : null
|
| 2479 |
+
// Intentionally single-shot on callOpen flip β we don't want this
|
| 2480 |
+
// to re-run on every message mutation during the call, otherwise
|
| 2481 |
+
// the guard would swallow the very replies it's supposed to let
|
| 2482 |
+
// through.
|
| 2483 |
+
// eslint-disable-next-line react-hooks/exhaustive-deps
|
| 2484 |
+
}, [callOpen])
|
| 2485 |
+
|
| 2486 |
// When a voice session is explicitly chosen from Session Hub, skip the auto-link resolve
|
| 2487 |
// to prevent the useEffect from overwriting the chosen session with the active one.
|
| 2488 |
const voiceSessionExplicitRef = useRef(false)
|
|
|
|
| 4157 |
[authHeaders, conversationId, messages, settings.backendUrl, settings.funMode, settingsDraft]
|
| 4158 |
)
|
| 4159 |
|
| 4160 |
+
// TTS for assistant responses (speak-once pattern). Fires when
|
| 4161 |
+
// the user is in Voice mode OR the CallOverlay is open. The
|
| 4162 |
+
// second branch matters in the chat-REST fallback path: when
|
| 4163 |
+
// VOICE_CALL_ENABLED is false server-side, the call runs through
|
| 4164 |
+
// the regular chat pipeline, assistant replies land in the
|
| 4165 |
+
// current ``messages`` array, and without this nothing speaks
|
| 4166 |
+
// them β the user hears silence during the call.
|
| 4167 |
+
//
|
| 4168 |
+
// SpeechService.speak + stripMarkdownForSpeech are the canonical
|
| 4169 |
+
// app-level TTS primitives; reusing them here keeps call-mode
|
| 4170 |
+
// TTS consistent with voice-mode TTS (same voice pick, same
|
| 4171 |
+
// markdown sanitation, same single-utterance dedupe via
|
| 4172 |
+
// lastSpokenMessageIdRef).
|
| 4173 |
useEffect(() => {
|
| 4174 |
+
if (mode !== 'voice' && !callOpen) return
|
|
|
|
| 4175 |
|
| 4176 |
const ttsEnabled = settingsDraft.ttsEnabled ?? true
|
| 4177 |
if (!ttsEnabled || !window.SpeechService) return
|
|
|
|
| 4184 |
lastMessage.role === 'assistant' &&
|
| 4185 |
!lastMessage.pending &&
|
| 4186 |
lastMessage.text &&
|
| 4187 |
+
!lastMessage.callMemory &&
|
| 4188 |
lastMessage.id !== lastSpokenMessageIdRef.current
|
| 4189 |
) {
|
| 4190 |
// Mark this message as spoken
|
|
|
|
| 4193 |
// Speak the assistant's response β strip markdown images/links so TTS
|
| 4194 |
// doesn't read raw URLs aloud. Images are shown visually instead.
|
| 4195 |
const speechText = stripMarkdownForSpeech(lastMessage.text)
|
| 4196 |
+
if (!speechText) return
|
| 4197 |
+
// During a call, route through the shared speakOwned lock so
|
| 4198 |
+
// this app-level path can't double-speak alongside the overlay's
|
| 4199 |
+
// fallback opener (both targeting the same assistant message).
|
| 4200 |
+
// Outside a call, keep the legacy SpeechService.speak path β
|
| 4201 |
+
// voice-mode has no peer TTS source so the lock would add noise
|
| 4202 |
+
// without benefit.
|
| 4203 |
+
if (callOpen && isCallFullDuplexEnabled()) {
|
| 4204 |
+
clog({ e: 'turn', action: 'assistant_in', route: 'chat_rest' })
|
| 4205 |
+
speakOwned('app-level', speechText)
|
| 4206 |
+
} else {
|
| 4207 |
+
window.SpeechService.speak(speechText)
|
| 4208 |
+
}
|
| 4209 |
}
|
| 4210 |
+
}, [messages, settingsDraft.ttsEnabled, mode, callOpen])
|
| 4211 |
|
| 4212 |
const uploadAndSend = useCallback(
|
| 4213 |
async (file: File) => {
|
|
|
|
| 4685 |
|
| 4686 |
{/* Top-right Project Indicator - Only shown in chat mode when a project is active */}
|
| 4687 |
{mode === 'chat' && (
|
| 4688 |
+
<header className="absolute top-0 left-0 right-0 pr-[9rem] pl-5 py-3 z-20 flex items-center justify-end gap-3 pointer-events-none">
|
| 4689 |
{/* Project Indicator */}
|
| 4690 |
{(() => {
|
| 4691 |
const currentProjectId = localStorage.getItem('homepilot_current_project')
|
|
|
|
| 5245 |
endRef={endRef}
|
| 5246 |
mode={mode}
|
| 5247 |
onNewConversation={onNewConversation}
|
| 5248 |
+
onStartCall={() => setCallOpen(true)}
|
| 5249 |
onRetryMessage={retryFailedMessage}
|
| 5250 |
chatSettings={chatSettings}
|
| 5251 |
onUpdateChatSettings={updateChatSettings}
|
|
|
|
| 5321 |
endRef={endRef}
|
| 5322 |
mode={mode}
|
| 5323 |
onNewConversation={onNewConversation}
|
| 5324 |
+
onStartCall={() => setCallOpen(true)}
|
| 5325 |
onRetryMessage={retryFailedMessage}
|
| 5326 |
chatSettings={chatSettings}
|
| 5327 |
onUpdateChatSettings={updateChatSettings}
|
|
|
|
| 5347 |
/>
|
| 5348 |
) : null}
|
| 5349 |
|
| 5350 |
+
{/* Call overlay β a distinct session layered over the chat.
|
| 5351 |
+
Open it with the π header icon; end it with the red button
|
| 5352 |
+
inside the overlay. Not the same as Voice mode. */}
|
| 5353 |
+
<CallOverlay
|
| 5354 |
+
open={callOpen}
|
| 5355 |
+
onClose={() => {
|
| 5356 |
+
setCallOpen(false)
|
| 5357 |
+
// Reset the skip flag once the overlay has closed so the
|
| 5358 |
+
// next fresh tap on π starts with the full dial phase.
|
| 5359 |
+
setCallSkipDialing(false)
|
| 5360 |
+
}}
|
| 5361 |
+
skipDialing={callSkipDialing}
|
| 5362 |
+
onEnded={(durationSec) => {
|
| 5363 |
+
// Append an inline PostCallCard to the chat stream. Captures
|
| 5364 |
+
// the slice of turns exchanged during the call β which the
|
| 5365 |
+
// user can expand inline from the card (no modal) β plus
|
| 5366 |
+
// the persona name + end-of-call timestamp.
|
| 5367 |
+
const endedAt = Date.now()
|
| 5368 |
+
const personaName = currentProject?.name || 'Assistant'
|
| 5369 |
+
const startIdx = callOpenMsgIndexRef.current
|
| 5370 |
+
// Build a transcript only when we have a valid open-time
|
| 5371 |
+
// snapshot AND new messages landed during the call.
|
| 5372 |
+
let transcript:
|
| 5373 |
+
| Array<{ who: 'user' | 'assistant'; text: string }>
|
| 5374 |
+
| undefined
|
| 5375 |
+
if (startIdx >= 0 && startIdx <= chatMessages.length) {
|
| 5376 |
+
const slice = chatMessages.slice(startIdx)
|
| 5377 |
+
const lines = slice
|
| 5378 |
+
.filter((m) => !m.callMemory && (m.text || '').trim())
|
| 5379 |
+
.map((m) => ({
|
| 5380 |
+
who: (m.role === 'user' ? 'user' : 'assistant') as
|
| 5381 |
+
| 'user' | 'assistant',
|
| 5382 |
+
text: m.text.trim(),
|
| 5383 |
+
}))
|
| 5384 |
+
if (lines.length > 0) transcript = lines
|
| 5385 |
+
}
|
| 5386 |
+
setChatMessages((prev) => [
|
| 5387 |
+
...prev,
|
| 5388 |
+
{
|
| 5389 |
+
id: `call-memory-${endedAt}`,
|
| 5390 |
+
role: 'assistant',
|
| 5391 |
+
text: '',
|
| 5392 |
+
callMemory: {
|
| 5393 |
+
durationSec,
|
| 5394 |
+
endedAt,
|
| 5395 |
+
personaName,
|
| 5396 |
+
transcript,
|
| 5397 |
+
},
|
| 5398 |
+
},
|
| 5399 |
+
])
|
| 5400 |
+
}}
|
| 5401 |
+
personaName={currentProject?.name || 'Assistant'}
|
| 5402 |
+
messages={chatMessages}
|
| 5403 |
+
avatarUrl={(() => {
|
| 5404 |
+
// Resolve the persona's thumbnail the same way
|
| 5405 |
+
// PersonaSettingsPanel does: prefer
|
| 5406 |
+
// persona_appearance.selected_thumb_filename, then
|
| 5407 |
+
// .selected_filename, wrapped in the /files/ auth URL.
|
| 5408 |
+
const pap = (currentProject?.persona_appearance as Record<string, unknown> | undefined) || {}
|
| 5409 |
+
const rel =
|
| 5410 |
+
(pap.selected_thumb_filename as string | undefined) ||
|
| 5411 |
+
(pap.selected_filename as string | undefined)
|
| 5412 |
+
if (!rel) return null
|
| 5413 |
+
const base = settings.backendUrl.replace(/\/+$/, '')
|
| 5414 |
+
const tok = localStorage.getItem('homepilot_auth_token') || ''
|
| 5415 |
+
const clean = rel.replace(/^\/+/, '')
|
| 5416 |
+
return `${base}/files/${clean}${tok ? `?token=${encodeURIComponent(tok)}` : ''}`
|
| 5417 |
+
})()}
|
| 5418 |
+
onSendText={(text) => sendTextOrIntent(text)}
|
| 5419 |
+
backend={{
|
| 5420 |
+
backendUrl: settings.backendUrl,
|
| 5421 |
+
authToken: localStorage.getItem('homepilot_auth_token'),
|
| 5422 |
+
conversationId: chatConversationId,
|
| 5423 |
+
personaId: currentProject?.persona_agent
|
| 5424 |
+
? (currentProject.persona_agent.id as string | undefined) || null
|
| 5425 |
+
: null,
|
| 5426 |
+
}}
|
| 5427 |
+
/>
|
| 5428 |
+
|
| 5429 |
+
{/* Previous floating "call ended" toast has been replaced by an
|
| 5430 |
+
inline CallMemoryCard rendered directly in the chat stream
|
| 5431 |
+
(see the messages.map branch on m.callMemory). Keeping this
|
| 5432 |
+
comment as a breadcrumb for reviewers. */}
|
| 5433 |
+
|
| 5434 |
+
|
| 5435 |
{/* Save as Persona Avatar modal (from AvatarStudio gallery) */}
|
| 5436 |
{saveAsPersonaData && (
|
| 5437 |
<SaveAsPersonaModal
|
frontend/src/ui/CallOverlay.tsx
ADDED
|
@@ -0,0 +1,1389 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* CallOverlay.tsx β TSX port of the frontend/src/ui/phone/ design set.
|
| 3 |
+
*
|
| 4 |
+
* Maps 1:1 onto the design-only mockups in `phone/`:
|
| 5 |
+
* tokens.jsx β HP_CALL + hpCallStateColor/Label (inlined below)
|
| 6 |
+
* icons.jsx β <Icon* /> components
|
| 7 |
+
* controls.jsx β <ControlBtn />
|
| 8 |
+
* avatar.jsx β <CallAvatar />
|
| 9 |
+
* waveform.jsx β <CallWaveform />
|
| 10 |
+
* call-modal.jsx β <CallModal /> + the ModalPresentation wrapper
|
| 11 |
+
*
|
| 12 |
+
* Layered as a distinct "call mode" on top of the chat β it is NOT
|
| 13 |
+
* the same thing as Voice mode:
|
| 14 |
+
* π€ Voice = input method inside the chat
|
| 15 |
+
* π Call = a separate immersive session (this component)
|
| 16 |
+
*
|
| 17 |
+
* States (mirrors the designer's set):
|
| 18 |
+
* connecting Β· listening Β· thinking Β· speaking Β· muted
|
| 19 |
+
* (ended is intentionally short-lived β the overlay fades out.)
|
| 20 |
+
*
|
| 21 |
+
* This pass wires the visual shell. Real STT/TTS plumbing from
|
| 22 |
+
* useVoiceController maps `listening/thinking/speaking` later.
|
| 23 |
+
*/
|
| 24 |
+
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
| 25 |
+
import { useVoiceController, type VoiceState } from './voice/useVoiceController'
|
| 26 |
+
import { useCallSession } from './call/useCallSession'
|
| 27 |
+
import Waveform, { type WaveformMode } from './phone/primitives/Waveform'
|
| 28 |
+
import Aura from './phone/primitives/Aura'
|
| 29 |
+
import { createStreamingTts, type StreamingTts } from './call/streamTts'
|
| 30 |
+
import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log'
|
| 31 |
+
import { useBargeInDetector } from './call/bargeIn'
|
| 32 |
+
|
| 33 |
+
export type CallState =
|
| 34 |
+
| 'dialing'
|
| 35 |
+
| 'connecting'
|
| 36 |
+
| 'listening'
|
| 37 |
+
| 'thinking'
|
| 38 |
+
| 'speaking'
|
| 39 |
+
| 'muted'
|
| 40 |
+
| 'ended'
|
| 41 |
+
|
| 42 |
+
// ββ Staged lifecycle timings. Numbers chosen so the user *feels* the
|
| 43 |
+
// handshake instead of getting a hard screen flip: dial (ringing) β
|
| 44 |
+
// connecting β listening (live). Dial is long enough for ~one full
|
| 45 |
+
// PSTN ringback cadence (2 s on / 4 s off in the US) so the user
|
| 46 |
+
// actually hears the phone "ring" before pickup.
|
| 47 |
+
const DIAL_MS = 2200 // π "callingβ¦" β pulse-ring phase + one PSTN ring
|
| 48 |
+
const CONNECT_MS = 500 // π΅ "connectingβ¦" β dots-pulse phase
|
| 49 |
+
const END_FADE_MS = 220 // π΄ end button β modal fade-out β onClose
|
| 50 |
+
|
| 51 |
+
// ββ Design tokens (ported from phone/tokens.jsx) ββββββββββββββββββ
|
| 52 |
+
|
| 53 |
+
const HP_CALL = {
|
| 54 |
+
font: 'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif',
|
| 55 |
+
fontTabular: 'ui-monospace, SFMono-Regular, "Roboto Mono", Menlo, monospace',
|
| 56 |
+
backdrop: 'rgba(5, 5, 6, 0.72)',
|
| 57 |
+
surface: '#0b0b0c',
|
| 58 |
+
surface2: '#121214',
|
| 59 |
+
border: 'rgba(255, 255, 255, 0.08)',
|
| 60 |
+
text: 'rgba(255, 255, 255, 0.92)',
|
| 61 |
+
text2: 'rgba(255, 255, 255, 0.55)',
|
| 62 |
+
text3: 'rgba(255, 255, 255, 0.35)',
|
| 63 |
+
accent: '#22d3ee',
|
| 64 |
+
stateListening: '#22d3ee',
|
| 65 |
+
stateThinking: '#a78bfa',
|
| 66 |
+
stateSpeaking: '#10b981',
|
| 67 |
+
stateError: '#f87171',
|
| 68 |
+
end: '#ef4444',
|
| 69 |
+
} as const
|
| 70 |
+
|
| 71 |
+
function hpCallStateColor(state: CallState): string {
|
| 72 |
+
switch (state) {
|
| 73 |
+
case 'listening': return HP_CALL.stateListening
|
| 74 |
+
case 'thinking': return HP_CALL.stateThinking
|
| 75 |
+
case 'speaking': return HP_CALL.stateSpeaking
|
| 76 |
+
case 'dialing':
|
| 77 |
+
case 'connecting': return HP_CALL.accent
|
| 78 |
+
case 'muted':
|
| 79 |
+
case 'ended': return HP_CALL.text3
|
| 80 |
+
default: return HP_CALL.text2
|
| 81 |
+
}
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
// Map the voice-controller's internal machine onto our UI machine.
|
| 85 |
+
// Returns null for states we don't want to reflect (OFF β overlay is
|
| 86 |
+
// open so "off" is not a meaningful UI frame; render last mapped).
|
| 87 |
+
function mapVoiceState(s: VoiceState): CallState | null {
|
| 88 |
+
switch (s) {
|
| 89 |
+
case 'LISTENING': return 'listening'
|
| 90 |
+
case 'THINKING': return 'thinking'
|
| 91 |
+
case 'SPEAKING': return 'speaking'
|
| 92 |
+
case 'IDLE': return 'listening'
|
| 93 |
+
case 'OFF': return null
|
| 94 |
+
default: return null
|
| 95 |
+
}
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
// Map the overlay's richer CallState onto the primitive Waveform's
|
| 99 |
+
// four-valued mode enum. The primitive doesn't know about dialing /
|
| 100 |
+
// connecting / thinking β those all fold to 'idle' (low ambient
|
| 101 |
+
// amplitude, faint colour); only the three audible states drive
|
| 102 |
+
// distinct waveform modes.
|
| 103 |
+
function waveformModeFromCallState(state: CallState): WaveformMode {
|
| 104 |
+
if (state === 'listening' || state === 'thinking') return 'listening'
|
| 105 |
+
if (state === 'speaking') return 'speaking'
|
| 106 |
+
if (state === 'muted') return 'muted'
|
| 107 |
+
return 'idle'
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
function hpCallStateLabel(state: CallState, personaName: string): string {
|
| 111 |
+
switch (state) {
|
| 112 |
+
case 'dialing': return `calling ${personaName.toLowerCase()}β¦`
|
| 113 |
+
case 'connecting': return 'connectingβ¦'
|
| 114 |
+
case 'listening': return 'listening'
|
| 115 |
+
case 'thinking': return 'thinking'
|
| 116 |
+
case 'speaking': return 'speaking'
|
| 117 |
+
case 'muted': return 'microphone off'
|
| 118 |
+
case 'ended': return 'call ended'
|
| 119 |
+
default: return ''
|
| 120 |
+
}
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
// ββ Icons (ported from phone/icons.jsx) βββββββββββββββββββββββββββ
|
| 124 |
+
|
| 125 |
+
type IconProps = { size?: number; color?: string }
|
| 126 |
+
const IconBase: React.FC<IconProps & { strokeWidth?: number; children: React.ReactNode }> = ({
|
| 127 |
+
size = 20, color = 'currentColor', strokeWidth = 1.75, children,
|
| 128 |
+
}) => (
|
| 129 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 130 |
+
stroke={color} strokeWidth={strokeWidth}
|
| 131 |
+
strokeLinecap="round" strokeLinejoin="round">
|
| 132 |
+
{children}
|
| 133 |
+
</svg>
|
| 134 |
+
)
|
| 135 |
+
const IconPhoneEnd: React.FC<IconProps> = (p) => (
|
| 136 |
+
<IconBase {...p} strokeWidth={1.9}>
|
| 137 |
+
<path d="M4 14c5-5 11-5 16 0l-2 2-3-1v-2a9 9 0 0 0-6 0v2l-3 1-2-2z" transform="rotate(135 12 12)" />
|
| 138 |
+
</IconBase>
|
| 139 |
+
)
|
| 140 |
+
const IconMic: React.FC<IconProps> = (p) => (
|
| 141 |
+
<IconBase {...p}>
|
| 142 |
+
<rect x="9" y="3" width="6" height="12" rx="3" />
|
| 143 |
+
<path d="M5 11a7 7 0 0 0 14 0M12 18v3" />
|
| 144 |
+
</IconBase>
|
| 145 |
+
)
|
| 146 |
+
const IconMicOff: React.FC<IconProps> = (p) => (
|
| 147 |
+
<IconBase {...p}>
|
| 148 |
+
<path d="M9 9V6a3 3 0 0 1 6 0v5m0 4a3 3 0 0 1-6 0" />
|
| 149 |
+
<path d="M5 11a7 7 0 0 0 11.5 5.3M19 11a7 7 0 0 1-.4 2.3M12 18v3" />
|
| 150 |
+
<path d="M3 3l18 18" />
|
| 151 |
+
</IconBase>
|
| 152 |
+
)
|
| 153 |
+
const IconChat: React.FC<IconProps> = (p) => (
|
| 154 |
+
<IconBase {...p}>
|
| 155 |
+
<path d="M4 5h16v11H9l-5 4z" />
|
| 156 |
+
</IconBase>
|
| 157 |
+
)
|
| 158 |
+
const IconBack: React.FC<IconProps> = (p) => (
|
| 159 |
+
<IconBase {...p} strokeWidth={1.9}>
|
| 160 |
+
<path d="M15 5l-7 7 7 7" />
|
| 161 |
+
</IconBase>
|
| 162 |
+
)
|
| 163 |
+
|
| 164 |
+
// ββ ControlBtn (ported from phone/controls.jsx) βββββββββββββββββββ
|
| 165 |
+
|
| 166 |
+
type Tone = 'neutral' | 'danger' | 'start' | 'accent'
|
| 167 |
+
const ControlBtn: React.FC<{
|
| 168 |
+
size?: number
|
| 169 |
+
tone?: Tone
|
| 170 |
+
active?: boolean
|
| 171 |
+
disabled?: boolean
|
| 172 |
+
label?: string
|
| 173 |
+
ariaLabel?: string
|
| 174 |
+
onClick?: () => void
|
| 175 |
+
children: React.ReactNode
|
| 176 |
+
}> = ({
|
| 177 |
+
size = 48, tone = 'neutral', active = false, disabled, label, ariaLabel, onClick, children,
|
| 178 |
+
}) => {
|
| 179 |
+
const bg =
|
| 180 |
+
tone === 'danger' ? HP_CALL.end :
|
| 181 |
+
tone === 'start' ? HP_CALL.stateSpeaking :
|
| 182 |
+
tone === 'accent' ? HP_CALL.accent :
|
| 183 |
+
active ? 'rgba(255,255,255,0.92)' : HP_CALL.surface2
|
| 184 |
+
const fg =
|
| 185 |
+
tone === 'danger' || tone === 'start' ? '#ffffff' :
|
| 186 |
+
tone === 'accent' ? '#052c33' :
|
| 187 |
+
active ? '#0b0b0c' : HP_CALL.text
|
| 188 |
+
const glow =
|
| 189 |
+
tone === 'danger' ? 'rgba(239, 68, 68, 0.45)' :
|
| 190 |
+
tone === 'start' ? 'rgba(16, 185, 129, 0.45)' :
|
| 191 |
+
tone === 'accent' ? 'rgba(34, 211, 238, 0.4)' :
|
| 192 |
+
'rgba(0, 0, 0, 0.3)'
|
| 193 |
+
const border = tone === 'neutral' && !active ? `1px solid ${HP_CALL.border}` : 'none'
|
| 194 |
+
|
| 195 |
+
return (
|
| 196 |
+
<div style={{
|
| 197 |
+
display: 'flex', flexDirection: 'column', alignItems: 'center', gap: 6,
|
| 198 |
+
opacity: disabled ? 0.45 : 1,
|
| 199 |
+
}}>
|
| 200 |
+
<button
|
| 201 |
+
type="button"
|
| 202 |
+
onClick={onClick}
|
| 203 |
+
disabled={disabled}
|
| 204 |
+
aria-label={ariaLabel || label || ''}
|
| 205 |
+
style={{
|
| 206 |
+
width: size, height: size, borderRadius: '50%',
|
| 207 |
+
background: bg, border, color: fg, padding: 0,
|
| 208 |
+
display: 'flex', alignItems: 'center', justifyContent: 'center',
|
| 209 |
+
cursor: disabled ? 'not-allowed' : 'pointer',
|
| 210 |
+
boxShadow:
|
| 211 |
+
tone === 'neutral' && !active
|
| 212 |
+
? 'inset 0 1px 0 rgba(255,255,255,0.04)'
|
| 213 |
+
: `0 8px 22px ${glow}, inset 0 1px 0 rgba(255,255,255,0.12)`,
|
| 214 |
+
transition: 'transform 80ms ease, background 120ms ease, box-shadow 120ms ease',
|
| 215 |
+
}}
|
| 216 |
+
>
|
| 217 |
+
{children}
|
| 218 |
+
</button>
|
| 219 |
+
{label ? (
|
| 220 |
+
<span style={{
|
| 221 |
+
fontFamily: HP_CALL.font, fontSize: 11, fontWeight: 500, letterSpacing: 0.2,
|
| 222 |
+
color: HP_CALL.text3, textTransform: 'lowercase',
|
| 223 |
+
}}>{label}</span>
|
| 224 |
+
) : null}
|
| 225 |
+
</div>
|
| 226 |
+
)
|
| 227 |
+
}
|
| 228 |
+
|
| 229 |
+
// ββ CallAvatar (ported from phone/avatar.jsx) βββββββββββββββββββββ
|
| 230 |
+
|
| 231 |
+
const CallAvatar: React.FC<{
|
| 232 |
+
size?: number
|
| 233 |
+
state: CallState
|
| 234 |
+
imageUrl?: string | null
|
| 235 |
+
accentColor?: string | null
|
| 236 |
+
}> = ({ size = 156, state, imageUrl = null, accentColor = null }) => {
|
| 237 |
+
const stateColor = hpCallStateColor(state)
|
| 238 |
+
const breathes =
|
| 239 |
+
state === 'listening' ||
|
| 240 |
+
state === 'connecting' ||
|
| 241 |
+
state === 'speaking' ||
|
| 242 |
+
state === 'dialing'
|
| 243 |
+
const showPulseRings = state === 'dialing'
|
| 244 |
+
const haloOuter = size + 28
|
| 245 |
+
// seed + hue derivation now live inside the Aura primitive
|
| 246 |
+
// (hashHue + moodOffset). This component only owns the halo +
|
| 247 |
+
// ring decorations that depend on CallState.
|
| 248 |
+
|
| 249 |
+
return (
|
| 250 |
+
<div style={{
|
| 251 |
+
position: 'relative', width: haloOuter, height: haloOuter,
|
| 252 |
+
display: 'flex', alignItems: 'center', justifyContent: 'center',
|
| 253 |
+
}}>
|
| 254 |
+
{/* Expanding pulse rings β only while dialing. Three rings on
|
| 255 |
+
staggered delays so the modal reads as "ringing", not just
|
| 256 |
+
"spinning up". */}
|
| 257 |
+
{showPulseRings && [0, 1, 2].map(i => (
|
| 258 |
+
<div key={i} aria-hidden="true" style={{
|
| 259 |
+
position: 'absolute', inset: 0, borderRadius: '50%',
|
| 260 |
+
border: `1.5px solid ${stateColor}`,
|
| 261 |
+
// Split to longhand so per-ring animationDelay doesn't race
|
| 262 |
+
// the shorthand reset β React warns when both are set
|
| 263 |
+
// (mixing shorthand + longhand produces inconsistent results
|
| 264 |
+
// across browsers).
|
| 265 |
+
animationName: 'hp-call-pulse-ring',
|
| 266 |
+
animationDuration: '1600ms',
|
| 267 |
+
animationTimingFunction: 'ease-out',
|
| 268 |
+
animationIterationCount: 'infinite',
|
| 269 |
+
animationDelay: `${i * 420}ms`,
|
| 270 |
+
opacity: 0,
|
| 271 |
+
}} />
|
| 272 |
+
))}
|
| 273 |
+
<div style={{
|
| 274 |
+
position: 'absolute', inset: 0, borderRadius: '50%',
|
| 275 |
+
background: `radial-gradient(circle, ${stateColor}55 0%, transparent 68%)`,
|
| 276 |
+
filter: 'blur(14px)',
|
| 277 |
+
opacity: breathes ? 0.8 : 0.35,
|
| 278 |
+
animation: breathes ? 'hp-halo-breathe 2s ease-in-out infinite' : 'none',
|
| 279 |
+
}} />
|
| 280 |
+
<div style={{
|
| 281 |
+
position: 'absolute', width: size + 10, height: size + 10,
|
| 282 |
+
borderRadius: '50%',
|
| 283 |
+
border: `2px solid ${stateColor}`,
|
| 284 |
+
opacity: state === 'ended' || state === 'muted' ? 0.25 : 0.92,
|
| 285 |
+
}} />
|
| 286 |
+
<div
|
| 287 |
+
style={{
|
| 288 |
+
// ``state === 'ended'`` filter stays on this wrapper so
|
| 289 |
+
// the Aura primitive stays state-agnostic while the
|
| 290 |
+
// overlay still gets a visible "call's over" cue.
|
| 291 |
+
filter: state === 'ended'
|
| 292 |
+
? 'grayscale(0.6) brightness(0.7)'
|
| 293 |
+
: 'none',
|
| 294 |
+
borderRadius: '50%',
|
| 295 |
+
}}
|
| 296 |
+
>
|
| 297 |
+
<Aura
|
| 298 |
+
seed={imageUrl || accentColor || 'homepilot'}
|
| 299 |
+
size={size}
|
| 300 |
+
photoUrl={imageUrl}
|
| 301 |
+
// Hue-drift polish is the only thing the Aura primitive
|
| 302 |
+
// animates internally. We disable it here β the overlay
|
| 303 |
+
// already owns the halo breath + dialing rings, and the
|
| 304 |
+
// combined motion was too busy on the call surface.
|
| 305 |
+
animated={false}
|
| 306 |
+
/>
|
| 307 |
+
</div>
|
| 308 |
+
</div>
|
| 309 |
+
)
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
// CallWaveform used to live here β extracted to
|
| 313 |
+
// frontend/src/ui/phone/primitives/Waveform.tsx. Consumed via the
|
| 314 |
+
// imported <Waveform /> component + waveformModeFromCallState
|
| 315 |
+
// mapper above.
|
| 316 |
+
|
| 317 |
+
// ββ CallModal (ported from phone/call-modal.jsx) ββββββββββββββββββ
|
| 318 |
+
|
| 319 |
+
interface CallModalProps {
|
| 320 |
+
state: CallState
|
| 321 |
+
personaName: string
|
| 322 |
+
imageUrl?: string | null
|
| 323 |
+
accentColor?: string | null
|
| 324 |
+
durationSec: number
|
| 325 |
+
onEnd: () => void
|
| 326 |
+
onToggleMute: () => void
|
| 327 |
+
onMinimize: () => void
|
| 328 |
+
onSwitchToChat?: () => void
|
| 329 |
+
/** Live 0..1 intensity. Drives the waveform bars per frame. */
|
| 330 |
+
intensityRef?: React.MutableRefObject<number>
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
const CallModal: React.FC<CallModalProps> = ({
|
| 334 |
+
state, personaName, imageUrl = null, accentColor = null,
|
| 335 |
+
durationSec, onEnd, onToggleMute, onMinimize, onSwitchToChat,
|
| 336 |
+
intensityRef,
|
| 337 |
+
}) => {
|
| 338 |
+
const stateColor = hpCallStateColor(state)
|
| 339 |
+
const stateLabel = hpCallStateLabel(state, personaName)
|
| 340 |
+
const mm = Math.floor(durationSec / 60).toString().padStart(2, '0')
|
| 341 |
+
const ss = (durationSec % 60).toString().padStart(2, '0')
|
| 342 |
+
const preConnect = state === 'dialing' || state === 'connecting'
|
| 343 |
+
const timer = preConnect ? 'β:β' : `${mm}:${ss}`
|
| 344 |
+
|
| 345 |
+
return (
|
| 346 |
+
<div style={{
|
| 347 |
+
width: 'min(420px, 92vw)',
|
| 348 |
+
padding: '18px 22px 26px',
|
| 349 |
+
borderRadius: 24,
|
| 350 |
+
background: HP_CALL.surface,
|
| 351 |
+
border: `1px solid ${HP_CALL.border}`,
|
| 352 |
+
boxShadow:
|
| 353 |
+
'0 30px 80px rgba(0,0,0,0.55), 0 0 0 1px rgba(0,0,0,0.4), inset 0 1px 0 rgba(255,255,255,0.04)',
|
| 354 |
+
position: 'relative', overflow: 'hidden',
|
| 355 |
+
fontFamily: HP_CALL.font, color: HP_CALL.text,
|
| 356 |
+
animation: 'hp-call-in 180ms ease-out',
|
| 357 |
+
}}>
|
| 358 |
+
<div style={{
|
| 359 |
+
position: 'absolute', inset: '-40% -10% auto -10%', height: '60%',
|
| 360 |
+
background: `radial-gradient(ellipse at 50% 100%, ${stateColor}26 0%, transparent 65%)`,
|
| 361 |
+
pointerEvents: 'none', opacity: state === 'muted' ? 0.1 : 0.35,
|
| 362 |
+
}} />
|
| 363 |
+
|
| 364 |
+
{/* Header row */}
|
| 365 |
+
<div style={{
|
| 366 |
+
position: 'relative', zIndex: 2,
|
| 367 |
+
display: 'flex', alignItems: 'center',
|
| 368 |
+
height: 32, marginBottom: 10,
|
| 369 |
+
}}>
|
| 370 |
+
<button
|
| 371 |
+
type="button"
|
| 372 |
+
onClick={onMinimize}
|
| 373 |
+
aria-label="Minimize call"
|
| 374 |
+
style={{
|
| 375 |
+
width: 32, height: 32, borderRadius: 10, padding: 0,
|
| 376 |
+
background: 'transparent', border: 'none', cursor: 'pointer',
|
| 377 |
+
color: HP_CALL.text2,
|
| 378 |
+
display: 'flex', alignItems: 'center', justifyContent: 'center',
|
| 379 |
+
}}
|
| 380 |
+
>
|
| 381 |
+
<IconBack size={18} color={HP_CALL.text2} />
|
| 382 |
+
</button>
|
| 383 |
+
<div style={{
|
| 384 |
+
position: 'absolute', left: 40, right: 40, textAlign: 'center',
|
| 385 |
+
fontSize: 16, fontWeight: 600, letterSpacing: -0.1,
|
| 386 |
+
color: HP_CALL.text,
|
| 387 |
+
whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis',
|
| 388 |
+
pointerEvents: 'none',
|
| 389 |
+
}}>{personaName}</div>
|
| 390 |
+
</div>
|
| 391 |
+
|
| 392 |
+
{/* Timer */}
|
| 393 |
+
<div style={{
|
| 394 |
+
position: 'relative', zIndex: 2, textAlign: 'center',
|
| 395 |
+
fontSize: 13, color: HP_CALL.text2,
|
| 396 |
+
fontFamily: HP_CALL.fontTabular,
|
| 397 |
+
fontVariantNumeric: 'tabular-nums',
|
| 398 |
+
marginBottom: 18,
|
| 399 |
+
}}>{timer}</div>
|
| 400 |
+
|
| 401 |
+
{/* Avatar + halo */}
|
| 402 |
+
<div style={{
|
| 403 |
+
position: 'relative', zIndex: 2,
|
| 404 |
+
display: 'flex', justifyContent: 'center', marginBottom: 18,
|
| 405 |
+
}}>
|
| 406 |
+
<CallAvatar size={156} state={state} imageUrl={imageUrl} accentColor={accentColor} />
|
| 407 |
+
</div>
|
| 408 |
+
|
| 409 |
+
{/* State label + waveform/dots */}
|
| 410 |
+
<div style={{
|
| 411 |
+
position: 'relative', zIndex: 2,
|
| 412 |
+
display: 'flex', flexDirection: 'column', alignItems: 'center',
|
| 413 |
+
gap: 10, marginBottom: 22, minHeight: 52,
|
| 414 |
+
}}>
|
| 415 |
+
{preConnect ? (
|
| 416 |
+
<div style={{ display: 'flex', gap: 5 }}>
|
| 417 |
+
{[0, 1, 2].map(i => (
|
| 418 |
+
<div key={i} style={{
|
| 419 |
+
width: 5, height: 5, borderRadius: '50%',
|
| 420 |
+
background: HP_CALL.text,
|
| 421 |
+
opacity: 0.35 + i * 0.2,
|
| 422 |
+
// Longhand β keeps per-dot animationDelay from being
|
| 423 |
+
// reset by the shorthand during React rerenders.
|
| 424 |
+
animationName: 'hp-dot-pulse',
|
| 425 |
+
animationDuration: state === 'dialing' ? '1.2s' : '0.8s',
|
| 426 |
+
animationTimingFunction: 'ease-in-out',
|
| 427 |
+
animationIterationCount: 'infinite',
|
| 428 |
+
animationDelay: `${i * 0.12}s`,
|
| 429 |
+
}} />
|
| 430 |
+
))}
|
| 431 |
+
</div>
|
| 432 |
+
) : (
|
| 433 |
+
<Waveform
|
| 434 |
+
bars={26}
|
| 435 |
+
height={24}
|
| 436 |
+
mode={waveformModeFromCallState(state)}
|
| 437 |
+
seed={personaName}
|
| 438 |
+
intensityRef={intensityRef}
|
| 439 |
+
/>
|
| 440 |
+
)}
|
| 441 |
+
<div style={{
|
| 442 |
+
fontSize: 14, fontWeight: 500, letterSpacing: 0.3,
|
| 443 |
+
color: stateColor, textTransform: 'lowercase',
|
| 444 |
+
}}>{stateLabel}</div>
|
| 445 |
+
</div>
|
| 446 |
+
|
| 447 |
+
{/* Three-button dock */}
|
| 448 |
+
<div style={{
|
| 449 |
+
position: 'relative', zIndex: 2,
|
| 450 |
+
display: 'flex', justifyContent: 'center', alignItems: 'center',
|
| 451 |
+
gap: 28,
|
| 452 |
+
}}>
|
| 453 |
+
<ControlBtn
|
| 454 |
+
size={56}
|
| 455 |
+
active={state === 'muted'}
|
| 456 |
+
ariaLabel={state === 'muted' ? 'Unmute microphone' : 'Mute microphone'}
|
| 457 |
+
onClick={onToggleMute}
|
| 458 |
+
disabled={state === 'dialing' || state === 'connecting' || state === 'ended'}
|
| 459 |
+
>
|
| 460 |
+
{state === 'muted'
|
| 461 |
+
? <IconMicOff size={22} color="#0b0b0c" />
|
| 462 |
+
: <IconMic size={22} />}
|
| 463 |
+
</ControlBtn>
|
| 464 |
+
|
| 465 |
+
<ControlBtn
|
| 466 |
+
size={72} tone="danger" ariaLabel="End call"
|
| 467 |
+
onClick={onEnd}
|
| 468 |
+
>
|
| 469 |
+
<IconPhoneEnd size={26} color="#ffffff" />
|
| 470 |
+
</ControlBtn>
|
| 471 |
+
|
| 472 |
+
<ControlBtn
|
| 473 |
+
size={56} ariaLabel="Switch to text chat"
|
| 474 |
+
onClick={onSwitchToChat}
|
| 475 |
+
disabled={!onSwitchToChat || state === 'dialing' || state === 'connecting' || state === 'ended'}
|
| 476 |
+
>
|
| 477 |
+
<IconChat size={22} />
|
| 478 |
+
</ControlBtn>
|
| 479 |
+
</div>
|
| 480 |
+
</div>
|
| 481 |
+
)
|
| 482 |
+
}
|
| 483 |
+
|
| 484 |
+
// ββ Public overlay component βββββββββββββββββββββββββββββββββββββ
|
| 485 |
+
|
| 486 |
+
export interface CallOverlayProps {
|
| 487 |
+
open: boolean
|
| 488 |
+
onClose: () => void
|
| 489 |
+
/** Optional: the persona the user is calling (display only). */
|
| 490 |
+
personaName?: string
|
| 491 |
+
/** Optional: avatar URL for the orb. */
|
| 492 |
+
avatarUrl?: string | null
|
| 493 |
+
/** Optional: per-persona accent for the fallback aura. */
|
| 494 |
+
accentColor?: string | null
|
| 495 |
+
/** Optional: minimize handler β reserved for the future PIP. */
|
| 496 |
+
onMinimize?: () => void
|
| 497 |
+
/** Optional: "Switch to text chat" handler. */
|
| 498 |
+
onSwitchToChat?: () => void
|
| 499 |
+
/** Skip the dial-ring phase. Used by "Speak again" re-entry so the
|
| 500 |
+
* user doesn't have to sit through "callingβ¦" twice in a row. */
|
| 501 |
+
skipDialing?: boolean
|
| 502 |
+
/** Fired when the call ends; receives the live-phase duration in
|
| 503 |
+
* seconds. Used by App.tsx to show the "call ended Β· Ns" toast. */
|
| 504 |
+
onEnded?: (durationSec: number) => void
|
| 505 |
+
/** Route a captured user utterance into the chat pipeline. When
|
| 506 |
+
* omitted the call runs in visual-only mode (no STT hook mounts).
|
| 507 |
+
* Only used as a fallback when the backend voice_call WS is
|
| 508 |
+
* unavailable (VOICE_CALL_ENABLED=false). */
|
| 509 |
+
onSendText?: (text: string) => void
|
| 510 |
+
/** When true the assistant is currently producing its reply (LLM
|
| 511 |
+
* is thinking). Maps CallState β 'thinking'. Reserved β the
|
| 512 |
+
* voice controller surfaces this itself once TTS events land. */
|
| 513 |
+
isAssistantThinking?: boolean
|
| 514 |
+
/** Backend config for the dedicated voice_call session. When
|
| 515 |
+
* supplied, the overlay will POST /v1/voice-call/sessions and
|
| 516 |
+
* open the WebSocket; if the backend responds with 404/501 we
|
| 517 |
+
* cleanly fall back to onSendText (chat REST). */
|
| 518 |
+
backend?: {
|
| 519 |
+
backendUrl: string
|
| 520 |
+
authToken: string | null
|
| 521 |
+
conversationId?: string | null
|
| 522 |
+
personaId?: string | null
|
| 523 |
+
}
|
| 524 |
+
/** Optional chat thread. When present AND the call runs in
|
| 525 |
+
* fallback mode (VOICE_CALL_ENABLED=false or unreachable),
|
| 526 |
+
* CallOverlay watches this array for new assistant replies
|
| 527 |
+
* that arrive DURING the call and speaks them via TTS β so
|
| 528 |
+
* the user hears the persona even without the voice_call WS.
|
| 529 |
+
*
|
| 530 |
+
* We also synthesize a short opener the moment the call
|
| 531 |
+
* connects in fallback mode, so the AI still "answers first"
|
| 532 |
+
* the way a real callee would. The baseline is captured at
|
| 533 |
+
* call-open so pre-existing chat history isn't re-spoken. */
|
| 534 |
+
messages?: ReadonlyArray<{
|
| 535 |
+
id: string
|
| 536 |
+
role: 'user' | 'assistant' | string
|
| 537 |
+
text: string
|
| 538 |
+
callMemory?: unknown
|
| 539 |
+
}>
|
| 540 |
+
}
|
| 541 |
+
|
| 542 |
+
// Outer shell β renders nothing when closed so the voice hook inside
|
| 543 |
+
// CallOverlayInner only mounts during an active call. This matters: the
|
| 544 |
+
// voice controller requests mic permission + starts the VAD as soon as
|
| 545 |
+
// it mounts. Keeping it gated behind `open` means the mic is only
|
| 546 |
+
// touched while the user is in a call.
|
| 547 |
+
export default function CallOverlay(props: CallOverlayProps) {
|
| 548 |
+
if (!props.open) return null
|
| 549 |
+
return <CallOverlayInner {...props} />
|
| 550 |
+
}
|
| 551 |
+
|
| 552 |
+
function CallOverlayInner({
|
| 553 |
+
onClose,
|
| 554 |
+
personaName = 'Assistant',
|
| 555 |
+
avatarUrl = null,
|
| 556 |
+
accentColor = null,
|
| 557 |
+
onMinimize,
|
| 558 |
+
onSwitchToChat,
|
| 559 |
+
skipDialing = false,
|
| 560 |
+
onEnded,
|
| 561 |
+
onSendText,
|
| 562 |
+
backend,
|
| 563 |
+
messages,
|
| 564 |
+
}: CallOverlayProps) {
|
| 565 |
+
const initial: CallState = skipDialing ? 'connecting' : 'dialing'
|
| 566 |
+
const [state, setState] = useState<CallState>(initial)
|
| 567 |
+
const [muted, setMuted] = useState(false)
|
| 568 |
+
const [durationSec, setDurationSec] = useState(0)
|
| 569 |
+
|
| 570 |
+
// Staged lifecycle on mount:
|
| 571 |
+
// dialing (DIAL_MS) β connecting (CONNECT_MS) β listening
|
| 572 |
+
// (or: connecting β listening when skipDialing).
|
| 573 |
+
useEffect(() => {
|
| 574 |
+
const timers: number[] = []
|
| 575 |
+
if (!skipDialing) {
|
| 576 |
+
timers.push(window.setTimeout(() => {
|
| 577 |
+
setState(s => (s === 'dialing' ? 'connecting' : s))
|
| 578 |
+
}, DIAL_MS))
|
| 579 |
+
}
|
| 580 |
+
timers.push(window.setTimeout(() => {
|
| 581 |
+
setState(s =>
|
| 582 |
+
(s === 'connecting' || s === 'dialing') ? 'listening' : s
|
| 583 |
+
)
|
| 584 |
+
}, (skipDialing ? 0 : DIAL_MS) + CONNECT_MS))
|
| 585 |
+
|
| 586 |
+
return () => { timers.forEach(t => window.clearTimeout(t)) }
|
| 587 |
+
}, [skipDialing])
|
| 588 |
+
|
| 589 |
+
// Live timer during active call (dialing + connecting don't count).
|
| 590 |
+
useEffect(() => {
|
| 591 |
+
if (state === 'dialing' || state === 'connecting' || state === 'ended') return
|
| 592 |
+
const iv = window.setInterval(() => setDurationSec((n) => n + 1), 1000)
|
| 593 |
+
return () => window.clearInterval(iv)
|
| 594 |
+
}, [state])
|
| 595 |
+
|
| 596 |
+
// Ringback tone β PSTN-standard two-frequency synthesis via
|
| 597 |
+
// WebAudio. US telephony uses 440 Hz + 480 Hz played together, 2 s
|
| 598 |
+
// on / 4 s off. We play one full on-phase (2 s) during dialing so
|
| 599 |
+
// the user hears a real phone-style ring before pickup. Nothing
|
| 600 |
+
// plays during connecting or live β a phone that keeps ringing
|
| 601 |
+
// after pickup is a broken phone.
|
| 602 |
+
useEffect(() => {
|
| 603 |
+
if (state !== 'dialing') return
|
| 604 |
+
const AC =
|
| 605 |
+
(window as unknown as { AudioContext?: typeof AudioContext }).AudioContext ||
|
| 606 |
+
(window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
|
| 607 |
+
if (!AC) return
|
| 608 |
+
let ctx: AudioContext | null = null
|
| 609 |
+
try { ctx = new AC() } catch { return }
|
| 610 |
+
|
| 611 |
+
const start = ctx.currentTime + 0.03
|
| 612 |
+
const durationSec = 2.0 // "on" portion of the 2-on-4-off cadence
|
| 613 |
+
const peakGain = 0.07 // loud enough to be heard, soft enough to sit
|
| 614 |
+
// under the persona's TTS once it lands
|
| 615 |
+
|
| 616 |
+
// 440 Hz + 480 Hz fundamentals β the exact CCITT-spec recipe for
|
| 617 |
+
// "ringing signal, North American" (Precise Tone Plan).
|
| 618 |
+
const freqs = [440, 480] as const
|
| 619 |
+
const oscs: OscillatorNode[] = []
|
| 620 |
+
const gain = ctx.createGain()
|
| 621 |
+
gain.gain.setValueAtTime(0, start)
|
| 622 |
+
gain.gain.linearRampToValueAtTime(peakGain, start + 0.08)
|
| 623 |
+
gain.gain.setValueAtTime(peakGain, start + durationSec - 0.2)
|
| 624 |
+
gain.gain.linearRampToValueAtTime(0, start + durationSec)
|
| 625 |
+
gain.connect(ctx.destination)
|
| 626 |
+
|
| 627 |
+
for (const f of freqs) {
|
| 628 |
+
const o = ctx.createOscillator()
|
| 629 |
+
o.type = 'sine'
|
| 630 |
+
o.frequency.setValueAtTime(f, start)
|
| 631 |
+
o.connect(gain)
|
| 632 |
+
o.start(start)
|
| 633 |
+
o.stop(start + durationSec + 0.05)
|
| 634 |
+
oscs.push(o)
|
| 635 |
+
}
|
| 636 |
+
|
| 637 |
+
return () => {
|
| 638 |
+
for (const o of oscs) { try { o.stop() } catch { /* ignore */ } }
|
| 639 |
+
try { ctx?.close() } catch { /* ignore */ }
|
| 640 |
+
}
|
| 641 |
+
}, [state])
|
| 642 |
+
|
| 643 |
+
const handleEnd = useCallback(() => {
|
| 644 |
+
setState('ended')
|
| 645 |
+
// If the backend session is live, send a graceful call.control
|
| 646 |
+
// 'end' envelope so the server-side ledger closes cleanly.
|
| 647 |
+
if (useBackendRef.current) {
|
| 648 |
+
try { sessionRef.current.end() } catch { /* ignore */ }
|
| 649 |
+
}
|
| 650 |
+
// Capture the live-phase duration NOW β durationSec is frozen at
|
| 651 |
+
// the last tick because the interval effect tears down on
|
| 652 |
+
// state==='ended'. Close after a short fade so the backdrop +
|
| 653 |
+
// modal get a chance to animate out.
|
| 654 |
+
const endedWith = durationSec
|
| 655 |
+
window.setTimeout(() => {
|
| 656 |
+
if (onEnded) onEnded(endedWith)
|
| 657 |
+
onClose()
|
| 658 |
+
}, END_FADE_MS)
|
| 659 |
+
}, [onClose, onEnded, durationSec])
|
| 660 |
+
|
| 661 |
+
// Esc ends.
|
| 662 |
+
useEffect(() => {
|
| 663 |
+
const onKey = (e: KeyboardEvent) => { if (e.key === 'Escape') handleEnd() }
|
| 664 |
+
window.addEventListener('keydown', onKey)
|
| 665 |
+
return () => window.removeEventListener('keydown', onKey)
|
| 666 |
+
}, [handleEnd])
|
| 667 |
+
|
| 668 |
+
// ββ Backend voice_call session (preferred transport) βββββββββββ
|
| 669 |
+
// When a `backend` prop is supplied we attempt the dedicated
|
| 670 |
+
// voice_call WebSocket. If the backend responds 404/501 we stay
|
| 671 |
+
// in 'unavailable' and the chat-REST fallback takes over below.
|
| 672 |
+
const session = useCallSession({
|
| 673 |
+
enabled: !!backend,
|
| 674 |
+
backendUrl: backend?.backendUrl ?? '',
|
| 675 |
+
authToken: backend?.authToken ?? null,
|
| 676 |
+
request: useMemo(() => ({
|
| 677 |
+
conversation_id: backend?.conversationId ?? null,
|
| 678 |
+
persona_id: backend?.personaId ?? null,
|
| 679 |
+
entry_mode: 'call' as const,
|
| 680 |
+
// Declare Phase 2/3 client capability. Server replies with
|
| 681 |
+
// its own capability advertisement; effective mode is the
|
| 682 |
+
// intersection (useCallSession reads capabilities from the
|
| 683 |
+
// session-create response).
|
| 684 |
+
device_info: {
|
| 685 |
+
tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
| 686 |
+
platform: navigator.platform,
|
| 687 |
+
streaming: true,
|
| 688 |
+
barge_in: true,
|
| 689 |
+
},
|
| 690 |
+
}), [backend?.conversationId, backend?.personaId]),
|
| 691 |
+
})
|
| 692 |
+
|
| 693 |
+
const useBackend =
|
| 694 |
+
!!backend &&
|
| 695 |
+
session.status !== 'unavailable' &&
|
| 696 |
+
session.status !== 'error'
|
| 697 |
+
|
| 698 |
+
// ββ "Listen first" gate β opener-TTS is playing βββββββββββββββ
|
| 699 |
+
// When the backend emits the curated opening greeting as a
|
| 700 |
+
// transcript.final right after call.state live, the client's
|
| 701 |
+
// microphone must stay CLOSED until the greeting has finished
|
| 702 |
+
// speaking. Otherwise the user's natural "hello?" mid-greeting
|
| 703 |
+
// triggers a barge-in and cuts the persona off β the opposite
|
| 704 |
+
// of the "AI answers first, user listens" product spec.
|
| 705 |
+
//
|
| 706 |
+
// openerDoneRef is a one-shot β once the first assistant
|
| 707 |
+
// utterance has finished, subsequent utterances (responses to
|
| 708 |
+
// user turns) do NOT relock the mic; the user can barge-in on
|
| 709 |
+
// those freely, which is the correct UX.
|
| 710 |
+
const [openerPlaying, setOpenerPlaying] = useState(false)
|
| 711 |
+
// ``connected`` is a pure derivation from ``state``; declared early
|
| 712 |
+
// so effects that fire before the setHandsFree gate (fallback opener
|
| 713 |
+
// below) can reference it without a forward-ref.
|
| 714 |
+
const connected =
|
| 715 |
+
state !== 'dialing' && state !== 'connecting' && state !== 'ended'
|
| 716 |
+
const openerDoneRef = useRef(false)
|
| 717 |
+
useEffect(() => {
|
| 718 |
+
if (session.status === 'creating') {
|
| 719 |
+
openerDoneRef.current = false
|
| 720 |
+
setOpenerPlaying(false)
|
| 721 |
+
}
|
| 722 |
+
}, [session.status])
|
| 723 |
+
|
| 724 |
+
// ββ Real voice pipeline ββββββββββββββββββββββββββββββββββββββββ
|
| 725 |
+
// STT capture β preferred transport. If the backend session is
|
| 726 |
+
// live we route transcripts through the voice_call WS; otherwise
|
| 727 |
+
// we fall back to onSendText (regular chat REST) so the overlay
|
| 728 |
+
// still works when VOICE_CALL_ENABLED is off server-side.
|
| 729 |
+
const onSendTextRef = useRef(onSendText)
|
| 730 |
+
useEffect(() => { onSendTextRef.current = onSendText }, [onSendText])
|
| 731 |
+
const sessionRef = useRef(session)
|
| 732 |
+
useEffect(() => { sessionRef.current = session }, [session])
|
| 733 |
+
const useBackendRef = useRef(useBackend)
|
| 734 |
+
useEffect(() => { useBackendRef.current = useBackend }, [useBackend])
|
| 735 |
+
|
| 736 |
+
const voice = useVoiceController((text: string) => {
|
| 737 |
+
clog({
|
| 738 |
+
e: 'turn',
|
| 739 |
+
action: 'user_out',
|
| 740 |
+
route: useBackendRef.current ? 'ws' : 'chat_rest',
|
| 741 |
+
})
|
| 742 |
+
if (useBackendRef.current) {
|
| 743 |
+
sessionRef.current.sendTranscript(text)
|
| 744 |
+
} else {
|
| 745 |
+
onSendTextRef.current?.(text)
|
| 746 |
+
}
|
| 747 |
+
})
|
| 748 |
+
|
| 749 |
+
// Speak assistant replies that land over the WS. (In the chat-REST
|
| 750 |
+
// fallback path, the existing assistant-message pipeline already
|
| 751 |
+
// triggers TTS β so this effect is a no-op there.)
|
| 752 |
+
//
|
| 753 |
+
// In STREAMING mode this stays a no-op for the assistant turn
|
| 754 |
+
// (partials feed streamTts below). It still fires for any
|
| 755 |
+
// transcript.final the server emits OUTSIDE a streamed turn β
|
| 756 |
+
// e.g. the opening greeting, which the current ws.py path sends
|
| 757 |
+
// as transcript.final regardless of streaming mode.
|
| 758 |
+
// Shared TTS helper β called by both the WS path AND the
|
| 759 |
+
// fallback-mode openers/replies below. The FIRST utterance on a
|
| 760 |
+
// mounted session is treated as the opener (locks the mic gate
|
| 761 |
+
// via openerPlaying / openerDoneRef); subsequent utterances play
|
| 762 |
+
// freely so the user can barge in.
|
| 763 |
+
|
| 764 |
+
// ββ Turn-lock state machine (full-duplex fix) βββββββββββββββββ
|
| 765 |
+
// The overlay owns a simple 3-state machine:
|
| 766 |
+
// 'ai' β AI is speaking; VAD events from the mic must be
|
| 767 |
+
// suppressed (the AI's own voice bleeding through the
|
| 768 |
+
// laptop speaker would otherwise trigger a false user
|
| 769 |
+
// turn β this is the "AI says yes, hears itself, then
|
| 770 |
+
// stops" bug reported in the log).
|
| 771 |
+
// 'user' β user is actively speaking (voice.state === LISTENING)
|
| 772 |
+
// 'idle' β nobody has the floor; VAD drives transitions normally
|
| 773 |
+
//
|
| 774 |
+
// Transitions fire setListeningSuppressed on the voice controller
|
| 775 |
+
// so suppression is a single source of truth. 300 ms release
|
| 776 |
+
// margin on 'ai β idle' lets echo tail die before we trust the
|
| 777 |
+
// mic again. Gated behind isCallFullDuplexEnabled() so the old
|
| 778 |
+
// behaviour is restorable via localStorage override.
|
| 779 |
+
type TurnLock = 'ai' | 'user' | 'idle'
|
| 780 |
+
const turnLockRef = useRef<TurnLock>('idle')
|
| 781 |
+
const turnUnlockTimerRef = useRef<number | null>(null)
|
| 782 |
+
const fullDuplexEnabledRef = useRef<boolean>(isCallFullDuplexEnabled())
|
| 783 |
+
|
| 784 |
+
const setTurnLock = useCallback((next: TurnLock, trigger: string) => {
|
| 785 |
+
if (!fullDuplexEnabledRef.current) return
|
| 786 |
+
const prev = turnLockRef.current
|
| 787 |
+
if (prev === next) return
|
| 788 |
+
turnLockRef.current = next
|
| 789 |
+
voice.setListeningSuppressed(next === 'ai', `turn_lock:${trigger}`)
|
| 790 |
+
clog({ e: 'state', from: prev, to: next, trigger })
|
| 791 |
+
}, [voice])
|
| 792 |
+
|
| 793 |
+
const releaseAiTurnWithMargin = useCallback((trigger: string) => {
|
| 794 |
+
if (turnUnlockTimerRef.current) {
|
| 795 |
+
window.clearTimeout(turnUnlockTimerRef.current)
|
| 796 |
+
turnUnlockTimerRef.current = null
|
| 797 |
+
}
|
| 798 |
+
turnUnlockTimerRef.current = window.setTimeout(() => {
|
| 799 |
+
setTurnLock('idle', `${trigger}:margin_release`)
|
| 800 |
+
turnUnlockTimerRef.current = null
|
| 801 |
+
}, 300)
|
| 802 |
+
}, [setTurnLock])
|
| 803 |
+
|
| 804 |
+
useEffect(() => {
|
| 805 |
+
return () => {
|
| 806 |
+
if (turnUnlockTimerRef.current) {
|
| 807 |
+
window.clearTimeout(turnUnlockTimerRef.current)
|
| 808 |
+
turnUnlockTimerRef.current = null
|
| 809 |
+
}
|
| 810 |
+
}
|
| 811 |
+
}, [])
|
| 812 |
+
|
| 813 |
+
const speakText = useCallback((text: string) => {
|
| 814 |
+
if (!text || !text.trim()) return
|
| 815 |
+
setTurnLock('ai', 'speak:start')
|
| 816 |
+
const isOpener = !openerDoneRef.current
|
| 817 |
+
if (isOpener) setOpenerPlaying(true)
|
| 818 |
+
const markDone = () => {
|
| 819 |
+
openerDoneRef.current = true
|
| 820 |
+
setOpenerPlaying(false)
|
| 821 |
+
// 300 ms margin before we flip the lock back to 'idle'. Echo
|
| 822 |
+
// tail can linger after TTS says "ended"; releasing immediately
|
| 823 |
+
// would let the mic re-open while our own voice is still in
|
| 824 |
+
// the air.
|
| 825 |
+
releaseAiTurnWithMargin('speak:end')
|
| 826 |
+
}
|
| 827 |
+
// Length-based estimate for the SpeechService path (the shim
|
| 828 |
+
// doesn't expose onend). ~12 chars/sec; floor 1.5 s so "Yes?"
|
| 829 |
+
// still gives the user a beat; ceiling 8 s so a runaway
|
| 830 |
+
// greeting can't lock the mic indefinitely.
|
| 831 |
+
const estimateSpeechMs = (t: string) =>
|
| 832 |
+
Math.max(1500, Math.min(8000, t.length * 80))
|
| 833 |
+
try {
|
| 834 |
+
// Prefer the owned TTS path when full-duplex is on. speakOwned
|
| 835 |
+
// handles both the window.SpeechService route AND the
|
| 836 |
+
// cross-source dedupe (see call/log.ts). Returns true iff this
|
| 837 |
+
// caller owns the utterance; false means another source
|
| 838 |
+
// (App-level TTS) was mid-speak β we fall through to the legacy
|
| 839 |
+
// speechSynthesis path unchanged so the overlay isn't silent.
|
| 840 |
+
if (fullDuplexEnabledRef.current) {
|
| 841 |
+
const spoke = speakOwned('overlay', text, {
|
| 842 |
+
onEnd: markDone,
|
| 843 |
+
onError: () => markDone(),
|
| 844 |
+
})
|
| 845 |
+
if (spoke) {
|
| 846 |
+
if (isOpener) window.setTimeout(markDone, estimateSpeechMs(text))
|
| 847 |
+
return
|
| 848 |
+
}
|
| 849 |
+
} else {
|
| 850 |
+
const w = window as unknown as {
|
| 851 |
+
SpeechService?: { speak?: (t: string) => void }
|
| 852 |
+
}
|
| 853 |
+
if (w.SpeechService?.speak) {
|
| 854 |
+
w.SpeechService.speak(text)
|
| 855 |
+
if (isOpener) window.setTimeout(markDone, estimateSpeechMs(text))
|
| 856 |
+
return
|
| 857 |
+
}
|
| 858 |
+
}
|
| 859 |
+
if ('speechSynthesis' in window) {
|
| 860 |
+
const utt = new SpeechSynthesisUtterance(text)
|
| 861 |
+
if (isOpener) {
|
| 862 |
+
utt.onend = markDone
|
| 863 |
+
utt.onerror = markDone
|
| 864 |
+
window.setTimeout(markDone, estimateSpeechMs(text))
|
| 865 |
+
}
|
| 866 |
+
window.speechSynthesis.speak(utt)
|
| 867 |
+
return
|
| 868 |
+
}
|
| 869 |
+
if (isOpener) markDone()
|
| 870 |
+
} catch {
|
| 871 |
+
if (isOpener) markDone()
|
| 872 |
+
}
|
| 873 |
+
}, [setTurnLock, releaseAiTurnWithMargin])
|
| 874 |
+
|
| 875 |
+
// WS path β speak every assistant transcript.final (the opening
|
| 876 |
+
// greeting from ws.py, and in unary mode every reply). In
|
| 877 |
+
// streaming mode, per-token partials go through streamTts below
|
| 878 |
+
// and THIS path fires only for transcript.final emitted outside
|
| 879 |
+
// a streamed turn β e.g. the opener itself.
|
| 880 |
+
useEffect(() => {
|
| 881 |
+
if (!useBackend) return
|
| 882 |
+
const unsub = session.onAssistantTranscript((p) => {
|
| 883 |
+
clog({ e: 'turn', action: 'assistant_in', route: 'ws' })
|
| 884 |
+
speakText(p.text)
|
| 885 |
+
})
|
| 886 |
+
return unsub
|
| 887 |
+
}, [useBackend, session, speakText])
|
| 888 |
+
|
| 889 |
+
// ββ Fallback mode (VOICE_CALL_ENABLED=false on backend) βββββββ
|
| 890 |
+
// When the backend voice_call route isn't mounted, the session
|
| 891 |
+
// POST 404s, useBackend stays false, and the STT pipeline routes
|
| 892 |
+
// through the regular chat-REST path (onSendText above). That
|
| 893 |
+
// works for text round-tripping β but without the hooks below,
|
| 894 |
+
// the assistant's replies land in chatMessages and are never
|
| 895 |
+
// spoken, so the user hears nothing.
|
| 896 |
+
//
|
| 897 |
+
// These two effects close that gap so the overlay is functional
|
| 898 |
+
// even with the WS disabled:
|
| 899 |
+
// (a) a synthetic opener fires when the call connects, so the
|
| 900 |
+
// AI still "answers first"
|
| 901 |
+
// (b) every new assistant message arriving in ``messages``
|
| 902 |
+
// during the call is spoken via the same TTS path the WS
|
| 903 |
+
// reply path uses
|
| 904 |
+
|
| 905 |
+
// (a) Fallback opener β once per session, when the line connects
|
| 906 |
+
// and we're running fallback mode, pick a short greeting and
|
| 907 |
+
// speak it. The greeting is marked as opener via speakText's
|
| 908 |
+
// first-utterance heuristic, so the mic gate stays closed
|
| 909 |
+
// exactly like the WS path.
|
| 910 |
+
const fallbackOpeners = useMemo(
|
| 911 |
+
() => [
|
| 912 |
+
'Hello?',
|
| 913 |
+
'Yes?',
|
| 914 |
+
`Hi, this is ${personaName}.`,
|
| 915 |
+
`${personaName} speaking.`,
|
| 916 |
+
`Hey β ${personaName}.`,
|
| 917 |
+
`Hi, ${personaName} here.`,
|
| 918 |
+
],
|
| 919 |
+
[personaName],
|
| 920 |
+
)
|
| 921 |
+
const fallbackOpenerFiredRef = useRef(false)
|
| 922 |
+
useEffect(() => {
|
| 923 |
+
if (useBackend) return
|
| 924 |
+
if (!connected) return
|
| 925 |
+
if (fallbackOpenerFiredRef.current) return
|
| 926 |
+
fallbackOpenerFiredRef.current = true
|
| 927 |
+
const g = fallbackOpeners[
|
| 928 |
+
Math.floor(Math.random() * fallbackOpeners.length)
|
| 929 |
+
]
|
| 930 |
+
speakText(g)
|
| 931 |
+
}, [useBackend, connected, fallbackOpeners, speakText])
|
| 932 |
+
|
| 933 |
+
// (b) Ongoing assistant replies in fallback mode are now spoken
|
| 934 |
+
// by the App-level TTS effect (App.tsx β gated on callOpen). That
|
| 935 |
+
// pipeline already handles stripMarkdownForSpeech + per-message
|
| 936 |
+
// dedupe via lastSpokenMessageIdRef, so we don't duplicate it
|
| 937 |
+
// here. The fallback OPENER above stays β it's synthesized in
|
| 938 |
+
// the overlay and never enters chatMessages, so the App-level
|
| 939 |
+
// effect can't produce it.
|
| 940 |
+
//
|
| 941 |
+
// ``messages`` prop is still accepted for future use (mid-call
|
| 942 |
+
// transcript viewer, etc.) but intentionally unused here β no
|
| 943 |
+
// TTS side-effect reads it.
|
| 944 |
+
|
| 945 |
+
// ββ Phase 2: streaming TTS pipeline βββββββββββββββββββββββββββ
|
| 946 |
+
// Holds one streamTts instance per mounted overlay. Stopped on
|
| 947 |
+
// unmount so a re-entry creates a fresh engine.
|
| 948 |
+
const ttsRef = useRef<StreamingTts | null>(null)
|
| 949 |
+
useEffect(() => {
|
| 950 |
+
if (!useBackend || !session.streamingNegotiated) {
|
| 951 |
+
ttsRef.current = null
|
| 952 |
+
return
|
| 953 |
+
}
|
| 954 |
+
const tts = createStreamingTts()
|
| 955 |
+
ttsRef.current = tts
|
| 956 |
+
return () => {
|
| 957 |
+
try { tts.stop() } catch { /* ignore */ }
|
| 958 |
+
ttsRef.current = null
|
| 959 |
+
}
|
| 960 |
+
}, [useBackend, session.streamingNegotiated])
|
| 961 |
+
|
| 962 |
+
// Track the currently-streaming turn_id so the barge-in path can
|
| 963 |
+
// address the right turn without racing a new one.
|
| 964 |
+
const currentTurnIdRef = useRef<string | null>(null)
|
| 965 |
+
|
| 966 |
+
// Partial deltas β streamTts.appendDelta. Each delta triggers
|
| 967 |
+
// sentence-boundary flushing inside the TTS engine; callers don't
|
| 968 |
+
// see the buffering.
|
| 969 |
+
useEffect(() => {
|
| 970 |
+
if (!useBackend || !session.streamingNegotiated) return
|
| 971 |
+
return session.onAssistantPartial((p) => {
|
| 972 |
+
currentTurnIdRef.current = p.turn_id
|
| 973 |
+
ttsRef.current?.appendDelta(p.delta)
|
| 974 |
+
})
|
| 975 |
+
}, [useBackend, session])
|
| 976 |
+
|
| 977 |
+
// Turn end β flush residual buffer + clear the active turn id.
|
| 978 |
+
useEffect(() => {
|
| 979 |
+
if (!useBackend || !session.streamingNegotiated) return
|
| 980 |
+
return session.onAssistantTurnEnd((p) => {
|
| 981 |
+
if (p.reason === 'cancelled' || p.reason === 'error') {
|
| 982 |
+
ttsRef.current?.stop()
|
| 983 |
+
} else {
|
| 984 |
+
ttsRef.current?.flush()
|
| 985 |
+
}
|
| 986 |
+
if (currentTurnIdRef.current === p.turn_id) {
|
| 987 |
+
currentTurnIdRef.current = null
|
| 988 |
+
}
|
| 989 |
+
})
|
| 990 |
+
}, [useBackend, session])
|
| 991 |
+
|
| 992 |
+
// Server-initiated cancel (barge-in ack) β stop TTS immediately.
|
| 993 |
+
// Must land inside ~50 ms of receipt (Β§ 5.3 contract).
|
| 994 |
+
useEffect(() => {
|
| 995 |
+
if (!useBackend || !session.streamingNegotiated) return
|
| 996 |
+
return session.onAssistantCancel((_p) => {
|
| 997 |
+
ttsRef.current?.stop()
|
| 998 |
+
})
|
| 999 |
+
}, [useBackend, session])
|
| 1000 |
+
|
| 1001 |
+
// ββ Phase 3: barge-in VAD tap βββββββββββββββββββββββββββββββββ
|
| 1002 |
+
// Audio level mirror β useVoiceController returns fresh values
|
| 1003 |
+
// every render; we stash the latest into a ref so the rAF loop
|
| 1004 |
+
// inside useBargeInDetector always sees the current value.
|
| 1005 |
+
const voiceLevelRef = useRef(0)
|
| 1006 |
+
useEffect(() => {
|
| 1007 |
+
voiceLevelRef.current = voice.audioLevel
|
| 1008 |
+
}, [voice.audioLevel])
|
| 1009 |
+
|
| 1010 |
+
// Re-evaluated each render; the detector's rAF loop reads its own
|
| 1011 |
+
// refs so identity changes don't thrash.
|
| 1012 |
+
const bargeInEnabled =
|
| 1013 |
+
useBackend &&
|
| 1014 |
+
session.bargeInNegotiated &&
|
| 1015 |
+
(ttsRef.current?.isSpeaking ?? false)
|
| 1016 |
+
|
| 1017 |
+
useBargeInDetector({
|
| 1018 |
+
audioLevelRef: voiceLevelRef,
|
| 1019 |
+
enabled: bargeInEnabled,
|
| 1020 |
+
onBargeIn: () => {
|
| 1021 |
+
const tid = currentTurnIdRef.current
|
| 1022 |
+
clog({ e: 'turn', action: 'barge_in', route: 'ws' })
|
| 1023 |
+
ttsRef.current?.stop()
|
| 1024 |
+
if (tid) session.sendBargeIn(tid)
|
| 1025 |
+
},
|
| 1026 |
+
})
|
| 1027 |
+
|
| 1028 |
+
// Call-center-style auto-greeting β the moment the backend session
|
| 1029 |
+
// goes live, prompt the persona to answer first (the way a real
|
| 1030 |
+
// person picks up the phone). We send a short synthetic transcript
|
| 1031 |
+
// the backend's persona_call phase machine will treat as turn 0 of
|
| 1032 |
+
// the opening sequence β it responds with a contextual "hello?" /
|
| 1033 |
+
// "hey, what's up?" in the persona's own voice.
|
| 1034 |
+
//
|
| 1035 |
+
// This is a one-shot per call session. `greetedRef` guards against
|
| 1036 |
+
// reconnect storms triggering a second "hello".
|
| 1037 |
+
const greetedRef = useRef(false)
|
| 1038 |
+
useEffect(() => {
|
| 1039 |
+
if (!useBackend) return
|
| 1040 |
+
if (session.callState !== 'live') return
|
| 1041 |
+
if (greetedRef.current) return
|
| 1042 |
+
greetedRef.current = true
|
| 1043 |
+
// One very short open-channel ping. persona_call reads this as the
|
| 1044 |
+
// caller-initiated summons in the Schegloff opening structure and
|
| 1045 |
+
// the persona answers.
|
| 1046 |
+
session.sendTranscript('[phone-call-open]')
|
| 1047 |
+
}, [useBackend, session])
|
| 1048 |
+
|
| 1049 |
+
// Reset the greet guard when a fresh session is (re)created.
|
| 1050 |
+
useEffect(() => {
|
| 1051 |
+
if (session.status === 'creating') greetedRef.current = false
|
| 1052 |
+
}, [session.status])
|
| 1053 |
+
|
| 1054 |
+
// Backchannel + filler events are short, low-volume TTS clips.
|
| 1055 |
+
// We route them through the same speak path; they're short enough
|
| 1056 |
+
// (one token) that interrupting a reply mid-stream is fine.
|
| 1057 |
+
useEffect(() => {
|
| 1058 |
+
if (!useBackend) return
|
| 1059 |
+
const unsub1 = session.onAssistantBackchannel((p) => {
|
| 1060 |
+
try { window.speechSynthesis?.speak?.(new SpeechSynthesisUtterance(p.token)) }
|
| 1061 |
+
catch { /* ignore */ }
|
| 1062 |
+
})
|
| 1063 |
+
const unsub2 = session.onAssistantFiller((p) => {
|
| 1064 |
+
try { window.speechSynthesis?.speak?.(new SpeechSynthesisUtterance(p.token)) }
|
| 1065 |
+
catch { /* ignore */ }
|
| 1066 |
+
})
|
| 1067 |
+
return () => { unsub1(); unsub2() }
|
| 1068 |
+
}, [useBackend, session])
|
| 1069 |
+
|
| 1070 |
+
// Enter hands-free once the line is "connected" AND the opener
|
| 1071 |
+
// has finished speaking. The second gate is the "listen first"
|
| 1072 |
+
// rule β the AI speaks its greeting and the user hears it out
|
| 1073 |
+
// before the mic opens; otherwise a natural early "hello?" trips
|
| 1074 |
+
// the barge-in detector and cuts the persona off mid-greeting.
|
| 1075 |
+
// ``connected`` is declared earlier (near state).
|
| 1076 |
+
//
|
| 1077 |
+
// This effect intentionally has NO cleanup that toggles handsFree β
|
| 1078 |
+
// the original cleanup raced against `openerPlaying` transitions
|
| 1079 |
+
// and re-render-driven re-runs (voice + setTurnLock change identity
|
| 1080 |
+
// on every render), which flipped handsFree OFF mid-call and left
|
| 1081 |
+
// state on OFF right when the user started speaking. The STT path
|
| 1082 |
+
// then dropped the VAD speech_start because state !== IDLE. Tear-
|
| 1083 |
+
// down lives in the dedicated unmount effect below.
|
| 1084 |
+
const voiceHandleRef = useRef(voice)
|
| 1085 |
+
useEffect(() => { voiceHandleRef.current = voice }, [voice])
|
| 1086 |
+
const setTurnLockRef = useRef(setTurnLock)
|
| 1087 |
+
useEffect(() => { setTurnLockRef.current = setTurnLock }, [setTurnLock])
|
| 1088 |
+
|
| 1089 |
+
useEffect(() => {
|
| 1090 |
+
if (!connected) return
|
| 1091 |
+
if (openerPlaying) return
|
| 1092 |
+
setTurnLockRef.current('idle', 'connected_open_mic')
|
| 1093 |
+
voiceHandleRef.current.setHandsFree(true)
|
| 1094 |
+
}, [connected, openerPlaying])
|
| 1095 |
+
|
| 1096 |
+
// Unmount-only cleanup. Guarantees the mic is released and the
|
| 1097 |
+
// turn-lock is reset when the overlay actually tears down (call
|
| 1098 |
+
// ended, user navigated away), without coupling that teardown to
|
| 1099 |
+
// the normal-effect re-run cycle above.
|
| 1100 |
+
useEffect(() => {
|
| 1101 |
+
return () => {
|
| 1102 |
+
try { voiceHandleRef.current.setHandsFree(false) } catch { /* ignore */ }
|
| 1103 |
+
try { setTurnLockRef.current('idle', 'cleanup') } catch { /* ignore */ }
|
| 1104 |
+
}
|
| 1105 |
+
}, [])
|
| 1106 |
+
|
| 1107 |
+
// Mirror the voice-controller's machine onto our UI state so the
|
| 1108 |
+
// halo + waveform + label actually reflect what's happening:
|
| 1109 |
+
// LISTENING β 'listening'
|
| 1110 |
+
// THINKING β 'thinking'
|
| 1111 |
+
// SPEAKING β 'speaking'
|
| 1112 |
+
// IDLE β 'listening' (ready, nothing to say yet)
|
| 1113 |
+
// OFF / muted β preserved
|
| 1114 |
+
useEffect(() => {
|
| 1115 |
+
if (!connected) return
|
| 1116 |
+
if (muted) return
|
| 1117 |
+
const mapped = mapVoiceState(voice.state)
|
| 1118 |
+
if (!mapped) return
|
| 1119 |
+
setState(prev => {
|
| 1120 |
+
if (prev === 'ended' || prev === 'muted') return prev
|
| 1121 |
+
return mapped
|
| 1122 |
+
})
|
| 1123 |
+
}, [voice.state, connected, muted])
|
| 1124 |
+
|
| 1125 |
+
// Mirror the voice-controller's LISTENING / THINKING onto the
|
| 1126 |
+
// turn-lock 'user' slot. The AI owns 'ai' (see speakText); VAD
|
| 1127 |
+
// drives 'user' only when the AI isn't already speaking. This is
|
| 1128 |
+
// the industry-standard "who has the floor" gate β with it, the
|
| 1129 |
+
// AI's TTS bleed through the speaker can't trip a false user turn
|
| 1130 |
+
// (the root cause of the "AI says yes, hears itself, stops" bug).
|
| 1131 |
+
useEffect(() => {
|
| 1132 |
+
if (!fullDuplexEnabledRef.current) return
|
| 1133 |
+
if (!connected) return
|
| 1134 |
+
if (muted) return
|
| 1135 |
+
if (voice.state === 'LISTENING' && turnLockRef.current === 'idle') {
|
| 1136 |
+
clog({ e: 'vad', action: 'speech_start', state: voice.state })
|
| 1137 |
+
setTurnLock('user', 'voice_listening')
|
| 1138 |
+
} else if (voice.state === 'THINKING' && turnLockRef.current === 'user') {
|
| 1139 |
+
clog({ e: 'vad', action: 'speech_end', state: voice.state })
|
| 1140 |
+
setTurnLock('idle', 'voice_thinking')
|
| 1141 |
+
}
|
| 1142 |
+
}, [voice.state, connected, muted, setTurnLock])
|
| 1143 |
+
|
| 1144 |
+
// While the opener TTS is playing, the UI label must read
|
| 1145 |
+
// 'speaking' β the persona is literally talking. Default state
|
| 1146 |
+
// transitions would land on 'listening' (post-connect default)
|
| 1147 |
+
// which is a lie: audio is going OUT of the speaker, not coming
|
| 1148 |
+
// IN to the mic. The mic is still closed (see the openerPlaying
|
| 1149 |
+
// gate on setHandsFree above), so voice.state stays OFF during
|
| 1150 |
+
// this window and the mirror effect above doesn't override us.
|
| 1151 |
+
useEffect(() => {
|
| 1152 |
+
if (!openerPlaying) return
|
| 1153 |
+
setState(prev =>
|
| 1154 |
+
(prev === 'ended' || prev === 'muted') ? prev : 'speaking',
|
| 1155 |
+
)
|
| 1156 |
+
}, [openerPlaying])
|
| 1157 |
+
|
| 1158 |
+
// ββ Live waveform intensity (0..1) βββββββββββββββββββββββββββββ
|
| 1159 |
+
// One rAF loop per state-group writes to a ref that CallWaveform
|
| 1160 |
+
// reads directly (no React re-renders). React only sees the state
|
| 1161 |
+
// label; the bars themselves are driven straight from the DOM.
|
| 1162 |
+
//
|
| 1163 |
+
// listening / muted β voice.audioLevel (real mic RMS)
|
| 1164 |
+
// speaking β synthesised speech envelope β browser TTS
|
| 1165 |
+
// audio isn't routable through an AnalyserNode,
|
| 1166 |
+
// so we generate a natural-sounding envelope
|
| 1167 |
+
// (sum of 3 sines + a small noise floor). The
|
| 1168 |
+
// visual cadence matches syllable rate + the
|
| 1169 |
+
// amplitude range matches what mic analysers
|
| 1170 |
+
// produce on voiced speech, so the persona's
|
| 1171 |
+
// bars and the user's bars "look like the
|
| 1172 |
+
// same kind of thing."
|
| 1173 |
+
// thinking β slow 0.15..0.25 drift β "processing"
|
| 1174 |
+
// dialing/connecting/ended β 0.05 floor β visible but quiet
|
| 1175 |
+
const intensityRef = useRef(0.05)
|
| 1176 |
+
const voiceRef = useRef(voice)
|
| 1177 |
+
useEffect(() => { voiceRef.current = voice }, [voice])
|
| 1178 |
+
|
| 1179 |
+
useEffect(() => {
|
| 1180 |
+
let raf = 0
|
| 1181 |
+
const start = performance.now()
|
| 1182 |
+
|
| 1183 |
+
if (state === 'listening' || state === 'muted') {
|
| 1184 |
+
const tick = () => {
|
| 1185 |
+
// Real mic level. useVoiceController already smooths with
|
| 1186 |
+
// an EMA internally; we pass it through unmodified.
|
| 1187 |
+
intensityRef.current = muted ? 0 : voiceRef.current.audioLevel
|
| 1188 |
+
raf = requestAnimationFrame(tick)
|
| 1189 |
+
}
|
| 1190 |
+
raf = requestAnimationFrame(tick)
|
| 1191 |
+
return () => cancelAnimationFrame(raf)
|
| 1192 |
+
}
|
| 1193 |
+
|
| 1194 |
+
if (state === 'speaking') {
|
| 1195 |
+
const tick = () => {
|
| 1196 |
+
const t = (performance.now() - start) / 1000
|
| 1197 |
+
// Three sines at the frequencies you'd see in actual
|
| 1198 |
+
// voiced speech analysis: ~2 Hz syllable rate, ~5 Hz
|
| 1199 |
+
// mid-band, ~10 Hz high-band flutter. A small noise floor
|
| 1200 |
+
// breaks the periodicity so bars don't pulse in lockstep.
|
| 1201 |
+
const v =
|
| 1202 |
+
0.45 +
|
| 1203 |
+
Math.sin(t * 2.3) * 0.22 +
|
| 1204 |
+
Math.sin(t * 5.7 + 1.3) * 0.16 +
|
| 1205 |
+
Math.sin(t * 11.1 + 2.7) * 0.09 +
|
| 1206 |
+
(Math.random() - 0.5) * 0.07
|
| 1207 |
+
intensityRef.current = Math.max(0, Math.min(1, v))
|
| 1208 |
+
raf = requestAnimationFrame(tick)
|
| 1209 |
+
}
|
| 1210 |
+
raf = requestAnimationFrame(tick)
|
| 1211 |
+
return () => cancelAnimationFrame(raf)
|
| 1212 |
+
}
|
| 1213 |
+
|
| 1214 |
+
if (state === 'thinking') {
|
| 1215 |
+
const tick = () => {
|
| 1216 |
+
const t = (performance.now() - start) / 1000
|
| 1217 |
+
intensityRef.current = 0.18 + Math.sin(t * 1.1) * 0.06
|
| 1218 |
+
raf = requestAnimationFrame(tick)
|
| 1219 |
+
}
|
| 1220 |
+
raf = requestAnimationFrame(tick)
|
| 1221 |
+
return () => cancelAnimationFrame(raf)
|
| 1222 |
+
}
|
| 1223 |
+
|
| 1224 |
+
// dialing / connecting / ended β static low floor.
|
| 1225 |
+
intensityRef.current = 0.05
|
| 1226 |
+
}, [state, muted])
|
| 1227 |
+
|
| 1228 |
+
const toggleMute = useCallback(() => {
|
| 1229 |
+
setMuted((m) => {
|
| 1230 |
+
const next = !m
|
| 1231 |
+
if (next) {
|
| 1232 |
+
voice.setHandsFree(false)
|
| 1233 |
+
setState('muted')
|
| 1234 |
+
} else {
|
| 1235 |
+
voice.setHandsFree(true)
|
| 1236 |
+
setState('listening')
|
| 1237 |
+
}
|
| 1238 |
+
if (useBackendRef.current) {
|
| 1239 |
+
try { sessionRef.current.sendUiState({ muted: next }) } catch { /* ignore */ }
|
| 1240 |
+
}
|
| 1241 |
+
return next
|
| 1242 |
+
})
|
| 1243 |
+
}, [voice])
|
| 1244 |
+
|
| 1245 |
+
const handleMinimize = useCallback(() => {
|
| 1246 |
+
if (onMinimize) onMinimize()
|
| 1247 |
+
else onClose()
|
| 1248 |
+
}, [onMinimize, onClose])
|
| 1249 |
+
|
| 1250 |
+
return (
|
| 1251 |
+
<div
|
| 1252 |
+
role="dialog"
|
| 1253 |
+
aria-modal="true"
|
| 1254 |
+
aria-label={`Call with ${personaName}`}
|
| 1255 |
+
className="fixed inset-0 z-[100] flex items-center justify-center"
|
| 1256 |
+
>
|
| 1257 |
+
<div
|
| 1258 |
+
className="absolute inset-0"
|
| 1259 |
+
style={{
|
| 1260 |
+
background: HP_CALL.backdrop,
|
| 1261 |
+
backdropFilter: 'blur(20px)',
|
| 1262 |
+
WebkitBackdropFilter: 'blur(20px)',
|
| 1263 |
+
transition: 'opacity 200ms ease',
|
| 1264 |
+
opacity: state === 'ended' ? 0 : 1,
|
| 1265 |
+
}}
|
| 1266 |
+
/>
|
| 1267 |
+
<div className="relative z-10">
|
| 1268 |
+
<CallModal
|
| 1269 |
+
state={state}
|
| 1270 |
+
personaName={personaName}
|
| 1271 |
+
imageUrl={avatarUrl}
|
| 1272 |
+
accentColor={accentColor}
|
| 1273 |
+
durationSec={durationSec}
|
| 1274 |
+
onEnd={handleEnd}
|
| 1275 |
+
onToggleMute={toggleMute}
|
| 1276 |
+
onMinimize={handleMinimize}
|
| 1277 |
+
onSwitchToChat={onSwitchToChat}
|
| 1278 |
+
intensityRef={intensityRef}
|
| 1279 |
+
/>
|
| 1280 |
+
</div>
|
| 1281 |
+
|
| 1282 |
+
<style>{`
|
| 1283 |
+
@keyframes hp-call-in {
|
| 1284 |
+
from { opacity: 0; transform: scale(0.95); }
|
| 1285 |
+
to { opacity: 1; transform: scale(1); }
|
| 1286 |
+
}
|
| 1287 |
+
@keyframes hp-halo-breathe {
|
| 1288 |
+
0%, 100% { opacity: 0.30; transform: scale(1); }
|
| 1289 |
+
50% { opacity: 0.85; transform: scale(1.06); }
|
| 1290 |
+
}
|
| 1291 |
+
@keyframes hp-dot-pulse {
|
| 1292 |
+
0%, 100% { transform: translateY(0); opacity: 0.35; }
|
| 1293 |
+
50% { transform: translateY(-3px); opacity: 1; }
|
| 1294 |
+
}
|
| 1295 |
+
@keyframes hp-call-pulse-ring {
|
| 1296 |
+
0% { transform: scale(1); opacity: 0.65; }
|
| 1297 |
+
80% { opacity: 0; }
|
| 1298 |
+
100% { transform: scale(1.55); opacity: 0; }
|
| 1299 |
+
}
|
| 1300 |
+
@keyframes hp-call-toast-in {
|
| 1301 |
+
from { opacity: 0; transform: translateY(8px); }
|
| 1302 |
+
to { opacity: 1; transform: translateY(0); }
|
| 1303 |
+
}
|
| 1304 |
+
`}</style>
|
| 1305 |
+
</div>
|
| 1306 |
+
)
|
| 1307 |
+
}
|
| 1308 |
+
|
| 1309 |
+
|
| 1310 |
+
// ββ Call-ended toast β the "you just hung up, want to call back?" chip
|
| 1311 |
+
|
| 1312 |
+
export interface CallEndedToastProps {
|
| 1313 |
+
/** Duration of the live phase (seconds). Rendered as "42s" / "3m 12s". */
|
| 1314 |
+
durationSec: number
|
| 1315 |
+
onSpeakAgain: () => void
|
| 1316 |
+
onDismiss: () => void
|
| 1317 |
+
}
|
| 1318 |
+
|
| 1319 |
+
function formatEndedDuration(s: number): string {
|
| 1320 |
+
if (s < 60) return `${s}s`
|
| 1321 |
+
const m = Math.floor(s / 60)
|
| 1322 |
+
const rest = s % 60
|
| 1323 |
+
return rest === 0 ? `${m}m` : `${m}m ${rest}s`
|
| 1324 |
+
}
|
| 1325 |
+
|
| 1326 |
+
export const CallEndedToast: React.FC<CallEndedToastProps> = ({
|
| 1327 |
+
durationSec, onSpeakAgain, onDismiss,
|
| 1328 |
+
}) => {
|
| 1329 |
+
return (
|
| 1330 |
+
<div
|
| 1331 |
+
role="status"
|
| 1332 |
+
aria-live="polite"
|
| 1333 |
+
className="fixed bottom-6 left-1/2 z-[95] -translate-x-1/2"
|
| 1334 |
+
style={{
|
| 1335 |
+
animation: 'hp-call-toast-in 220ms ease-out',
|
| 1336 |
+
fontFamily: HP_CALL.font,
|
| 1337 |
+
}}
|
| 1338 |
+
>
|
| 1339 |
+
<div style={{
|
| 1340 |
+
display: 'flex', alignItems: 'center', gap: 12,
|
| 1341 |
+
padding: '8px 10px 8px 14px',
|
| 1342 |
+
borderRadius: 9999,
|
| 1343 |
+
background: HP_CALL.surface,
|
| 1344 |
+
border: `1px solid ${HP_CALL.border}`,
|
| 1345 |
+
boxShadow: '0 12px 32px rgba(0,0,0,0.45), inset 0 1px 0 rgba(255,255,255,0.04)',
|
| 1346 |
+
color: HP_CALL.text,
|
| 1347 |
+
}}>
|
| 1348 |
+
<span style={{
|
| 1349 |
+
display: 'inline-flex', alignItems: 'center', gap: 8,
|
| 1350 |
+
fontSize: 13, fontWeight: 500, color: HP_CALL.text2,
|
| 1351 |
+
}}>
|
| 1352 |
+
<span style={{
|
| 1353 |
+
width: 6, height: 6, borderRadius: '50%',
|
| 1354 |
+
background: HP_CALL.text3,
|
| 1355 |
+
}} />
|
| 1356 |
+
Call ended Β· <span style={{
|
| 1357 |
+
color: HP_CALL.text,
|
| 1358 |
+
fontFamily: HP_CALL.fontTabular,
|
| 1359 |
+
fontVariantNumeric: 'tabular-nums',
|
| 1360 |
+
}}>{formatEndedDuration(durationSec)}</span>
|
| 1361 |
+
</span>
|
| 1362 |
+
<button
|
| 1363 |
+
type="button"
|
| 1364 |
+
onClick={onSpeakAgain}
|
| 1365 |
+
style={{
|
| 1366 |
+
height: 30, padding: '0 14px', borderRadius: 9999,
|
| 1367 |
+
background: HP_CALL.accent,
|
| 1368 |
+
color: '#052c33', border: 'none', cursor: 'pointer',
|
| 1369 |
+
fontSize: 12, fontWeight: 600, letterSpacing: 0.2,
|
| 1370 |
+
}}
|
| 1371 |
+
>
|
| 1372 |
+
Speak again
|
| 1373 |
+
</button>
|
| 1374 |
+
<button
|
| 1375 |
+
type="button"
|
| 1376 |
+
onClick={onDismiss}
|
| 1377 |
+
aria-label="Dismiss"
|
| 1378 |
+
style={{
|
| 1379 |
+
width: 28, height: 28, borderRadius: 9999,
|
| 1380 |
+
background: 'transparent', border: 'none', cursor: 'pointer',
|
| 1381 |
+
color: HP_CALL.text3, fontSize: 18, lineHeight: 1, padding: 0,
|
| 1382 |
+
}}
|
| 1383 |
+
>
|
| 1384 |
+
Γ
|
| 1385 |
+
</button>
|
| 1386 |
+
</div>
|
| 1387 |
+
</div>
|
| 1388 |
+
)
|
| 1389 |
+
}
|
frontend/src/ui/CreatorStudioEditor.tsx
CHANGED
|
@@ -1847,6 +1847,12 @@ export function CreatorStudioEditor({
|
|
| 1847 |
backendUrl={backendUrl}
|
| 1848 |
videoId={projectId}
|
| 1849 |
videoTitle={project?.title}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1850 |
disabledReason={
|
| 1851 |
scenes.length === 0
|
| 1852 |
? "Add at least one scene before exporting."
|
|
|
|
| 1847 |
backendUrl={backendUrl}
|
| 1848 |
videoId={projectId}
|
| 1849 |
videoTitle={project?.title}
|
| 1850 |
+
scenes={scenes.map((s) => ({
|
| 1851 |
+
id: s.id,
|
| 1852 |
+
idx: s.idx,
|
| 1853 |
+
narration: s.narration || "",
|
| 1854 |
+
audioUrl: s.audioUrl || null,
|
| 1855 |
+
}))}
|
| 1856 |
disabledReason={
|
| 1857 |
scenes.length === 0
|
| 1858 |
? "Add at least one scene before exporting."
|
frontend/src/ui/SettingsPanel.tsx
CHANGED
|
@@ -1,6 +1,15 @@
|
|
| 1 |
import React, { useEffect, useMemo, useState } from "react";
|
| 2 |
import OllamaHealthBanner from "./OllamaHealthBanner";
|
| 3 |
import ProfileSettingsModal from "./ProfileSettingsModal";
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4 |
import { resolveBackendUrl } from "./lib/backendUrl";
|
| 5 |
import {
|
| 6 |
getModelSettings,
|
|
@@ -208,6 +217,12 @@ export default function SettingsPanel({
|
|
| 208 |
|
| 209 |
// Voice state
|
| 210 |
const [availableVoices, setAvailableVoices] = useState<SpeechSynthesisVoice[]>([]);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 211 |
const [voicesInitialized, setVoicesInitialized] = useState(false);
|
| 212 |
|
| 213 |
// Video presets state (fetched from backend based on selected model)
|
|
@@ -1084,27 +1099,40 @@ export default function SettingsPanel({
|
|
| 1084 |
<span className="text-xs text-white">Enable Text-to-Speech (TTS)</span>
|
| 1085 |
</label>
|
| 1086 |
|
| 1087 |
-
{/* Voice Selection
|
| 1088 |
-
|
| 1089 |
-
|
| 1090 |
-
|
| 1091 |
-
|
| 1092 |
-
|
| 1093 |
-
|
| 1094 |
-
|
| 1095 |
-
|
| 1096 |
-
|
| 1097 |
-
|
| 1098 |
-
|
| 1099 |
-
</option>
|
| 1100 |
-
|
| 1101 |
-
|
| 1102 |
-
|
| 1103 |
-
|
| 1104 |
-
|
| 1105 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1106 |
</div>
|
| 1107 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1108 |
</div>
|
| 1109 |
</div>
|
| 1110 |
|
|
|
|
| 1 |
import React, { useEffect, useMemo, useState } from "react";
|
| 2 |
import OllamaHealthBanner from "./OllamaHealthBanner";
|
| 3 |
import ProfileSettingsModal from "./ProfileSettingsModal";
|
| 4 |
+
import TtsEngineSection from "./components/TtsEngineSection";
|
| 5 |
+
// Side-effect import: registers the bundled TTS providers (web-speech-api,
|
| 6 |
+
// piper-wasm). Importing here guarantees the registry is populated the
|
| 7 |
+
// first time the Settings panel mounts, before TtsEngineSection reads it.
|
| 8 |
+
import "./tts";
|
| 9 |
+
import {
|
| 10 |
+
getActiveTtsEngineId,
|
| 11 |
+
onActiveTtsEngineChange,
|
| 12 |
+
} from "./tts";
|
| 13 |
import { resolveBackendUrl } from "./lib/backendUrl";
|
| 14 |
import {
|
| 15 |
getModelSettings,
|
|
|
|
| 217 |
|
| 218 |
// Voice state
|
| 219 |
const [availableVoices, setAvailableVoices] = useState<SpeechSynthesisVoice[]>([]);
|
| 220 |
+
// Track the active TTS engine so we can hide the legacy "Assistant
|
| 221 |
+
// Voice" select when the user picks a non-default engine (Piper etc.).
|
| 222 |
+
// Purely cosmetic: the underlying value.selectedVoice state is not
|
| 223 |
+
// touched β only the control rendering it.
|
| 224 |
+
const [ttsEngineId, setTtsEngineId] = useState<string>(() => getActiveTtsEngineId());
|
| 225 |
+
useEffect(() => onActiveTtsEngineChange(setTtsEngineId), []);
|
| 226 |
const [voicesInitialized, setVoicesInitialized] = useState(false);
|
| 227 |
|
| 228 |
// Video presets state (fetched from backend based on selected model)
|
|
|
|
| 1099 |
<span className="text-xs text-white">Enable Text-to-Speech (TTS)</span>
|
| 1100 |
</label>
|
| 1101 |
|
| 1102 |
+
{/* Voice Selection β only shown for the default engine. When
|
| 1103 |
+
the user picks a non-default engine (Piper etc.) the engine
|
| 1104 |
+
owns its own voice picker below, and showing two would be
|
| 1105 |
+
confusing duplication. */}
|
| 1106 |
+
{ttsEngineId === 'web-speech-api' && (
|
| 1107 |
+
<div>
|
| 1108 |
+
<label className="block text-[10px] text-white/50 mb-2">Assistant Voice</label>
|
| 1109 |
+
<select
|
| 1110 |
+
className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
|
| 1111 |
+
value={value.selectedVoice ?? ''}
|
| 1112 |
+
onChange={(e) => onChangeDraft({ ...value, selectedVoice: e.target.value })}
|
| 1113 |
+
>
|
| 1114 |
+
<option value="">System Default</option>
|
| 1115 |
+
{availableVoices.map((voice) => (
|
| 1116 |
+
<option key={voice.voiceURI} value={voice.name}>
|
| 1117 |
+
{voice.name} ({voice.lang})
|
| 1118 |
+
</option>
|
| 1119 |
+
))}
|
| 1120 |
+
</select>
|
| 1121 |
+
<div className="mt-1 text-[10px] text-white/40">
|
| 1122 |
+
{availableVoices.length > 0
|
| 1123 |
+
? `Choose from ${availableVoices.length} available voices`
|
| 1124 |
+
: 'Loading voices...'}
|
| 1125 |
+
</div>
|
| 1126 |
</div>
|
| 1127 |
+
)}
|
| 1128 |
+
|
| 1129 |
+
{/* TTS Engine picker β additive plugin system. The existing
|
| 1130 |
+
System Voice selector above stays as the default path; this
|
| 1131 |
+
section lets the user opt into alternative engines (Piper
|
| 1132 |
+
WASM today; more can register themselves without touching
|
| 1133 |
+
this file). Schema-driven controls so future engines
|
| 1134 |
+
require zero Settings edits. */}
|
| 1135 |
+
<TtsEngineSection systemVoices={availableVoices} />
|
| 1136 |
</div>
|
| 1137 |
</div>
|
| 1138 |
|
frontend/src/ui/call/bargeIn.test.ts
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Unit tests for the pure detector in bargeIn.ts. Covers the exact
|
| 3 |
+
* Β§ 6 invariants (single-spike rejection, sustain gate, refractory).
|
| 4 |
+
* The hook form is intentionally not tested here β it's a thin
|
| 5 |
+
* wrapper around the detector + rAF; coverage lives in the overlay
|
| 6 |
+
* integration path.
|
| 7 |
+
*/
|
| 8 |
+
import { describe, it, expect } from 'vitest'
|
| 9 |
+
import { detectBargeIn } from './bargeIn'
|
| 10 |
+
|
| 11 |
+
const OPTS = {
|
| 12 |
+
threshold: 0.08,
|
| 13 |
+
minSustainMs: 80,
|
| 14 |
+
minRefractoryMs: 400,
|
| 15 |
+
} as const
|
| 16 |
+
|
| 17 |
+
function genConstant(level: number, durMs: number, stepMs = 16) {
|
| 18 |
+
const out: Array<{ tMs: number; level: number }> = []
|
| 19 |
+
for (let t = 0; t <= durMs; t += stepMs) {
|
| 20 |
+
out.push({ tMs: t, level })
|
| 21 |
+
}
|
| 22 |
+
return out
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
describe('detectBargeIn', () => {
|
| 26 |
+
it('fires once after sustained speech above threshold', () => {
|
| 27 |
+
// Steady high level for 200 ms β comfortably over the 80 ms sustain.
|
| 28 |
+
const events = detectBargeIn(genConstant(0.2, 200), OPTS)
|
| 29 |
+
expect(events.length).toBe(1)
|
| 30 |
+
expect(events[0]).toBeGreaterThanOrEqual(OPTS.minSustainMs)
|
| 31 |
+
})
|
| 32 |
+
|
| 33 |
+
it('does NOT fire on a single spike below sustain', () => {
|
| 34 |
+
// One 16 ms spike followed by silence. Below 80 ms sustain β drops.
|
| 35 |
+
const samples = [
|
| 36 |
+
{ tMs: 0, level: 0.0 },
|
| 37 |
+
{ tMs: 16, level: 0.2 },
|
| 38 |
+
{ tMs: 32, level: 0.0 },
|
| 39 |
+
{ tMs: 48, level: 0.0 },
|
| 40 |
+
{ tMs: 64, level: 0.0 },
|
| 41 |
+
]
|
| 42 |
+
expect(detectBargeIn(samples, OPTS)).toEqual([])
|
| 43 |
+
})
|
| 44 |
+
|
| 45 |
+
it('does NOT fire when level never exceeds threshold', () => {
|
| 46 |
+
const events = detectBargeIn(genConstant(0.04, 400), OPTS)
|
| 47 |
+
expect(events).toEqual([])
|
| 48 |
+
})
|
| 49 |
+
|
| 50 |
+
it('honours the refractory window (no double-fire during sustained speech)', () => {
|
| 51 |
+
// 1500 ms of steady speech. Without refractory we'd fire every
|
| 52 |
+
// tick after 80 ms. With refractory we fire at most once per
|
| 53 |
+
// minRefractoryMs (~400 ms).
|
| 54 |
+
const events = detectBargeIn(genConstant(0.2, 1500), OPTS)
|
| 55 |
+
expect(events.length).toBeGreaterThan(0)
|
| 56 |
+
// At 16 ms steps and 400 ms refractory: expect ~3 events.
|
| 57 |
+
expect(events.length).toBeLessThanOrEqual(4)
|
| 58 |
+
// Consecutive events must be at least refractory-ms apart.
|
| 59 |
+
for (let i = 1; i < events.length; i++) {
|
| 60 |
+
expect(events[i] - events[i - 1]).toBeGreaterThanOrEqual(
|
| 61 |
+
OPTS.minRefractoryMs,
|
| 62 |
+
)
|
| 63 |
+
}
|
| 64 |
+
})
|
| 65 |
+
|
| 66 |
+
it('resets the sustain counter after a dip below threshold', () => {
|
| 67 |
+
// 50 ms above, 30 ms below, 100 ms above. First run is too short;
|
| 68 |
+
// second run also too short on its own because counter reset.
|
| 69 |
+
const samples = [
|
| 70 |
+
...Array.from({ length: 4 }, (_, i) => ({
|
| 71 |
+
tMs: i * 16, level: 0.2,
|
| 72 |
+
})),
|
| 73 |
+
{ tMs: 64, level: 0.0 },
|
| 74 |
+
{ tMs: 80, level: 0.0 },
|
| 75 |
+
...Array.from({ length: 4 }, (_, i) => ({
|
| 76 |
+
tMs: 96 + i * 16, level: 0.2,
|
| 77 |
+
})),
|
| 78 |
+
]
|
| 79 |
+
expect(detectBargeIn(samples, OPTS)).toEqual([])
|
| 80 |
+
})
|
| 81 |
+
})
|
frontend/src/ui/call/bargeIn.ts
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* bargeIn.ts β VAD-tap hook that detects user speech-start while
|
| 3 |
+
* the assistant's TTS is playing, so the client can interrupt the
|
| 4 |
+
* persona the way a person would.
|
| 5 |
+
*
|
| 6 |
+
* Design (Β§ 5.4 of docs/analysis/voice-call-streaming-design.md):
|
| 7 |
+
*
|
| 8 |
+
* - Reads the EMA-smoothed mic level that useVoiceController
|
| 9 |
+
* already exposes (audioLevelRef), sampled at rAF cadence.
|
| 10 |
+
* - Fires onBargeIn when the level stays above ``threshold`` for
|
| 11 |
+
* at least ``minSustainMs``. Sustain protects against a
|
| 12 |
+
* single spike (keyboard click, cough) cutting the persona off.
|
| 13 |
+
* - Only enabled when the overlay says it should be β i.e. the
|
| 14 |
+
* TTS is actually speaking AND the session negotiated streaming.
|
| 15 |
+
* Outside that window it's a pure no-op.
|
| 16 |
+
*
|
| 17 |
+
* The threshold here (default 0.08) is deliberately HIGHER than the
|
| 18 |
+
* baseline VAD threshold (~0.035) that useVoiceController uses for
|
| 19 |
+
* end-of-user-turn silence detection. Reason: during TTS playback a
|
| 20 |
+
* fraction of the persona's own audio returns into the mic on most
|
| 21 |
+
* laptops; a low threshold trips on that bleed and generates
|
| 22 |
+
* phantom barge-ins. 0.08 is above typical TTS bleed but well below
|
| 23 |
+
* normal voiced-speech peaks.
|
| 24 |
+
*/
|
| 25 |
+
|
| 26 |
+
import { useEffect, useRef } from 'react'
|
| 27 |
+
|
| 28 |
+
export interface UseBargeInDetectorArgs {
|
| 29 |
+
/** Shared ref mutated at rAF cadence by the parent β typically
|
| 30 |
+
* ``voice.audioLevel`` mirrored into a ref so we avoid stale
|
| 31 |
+
* reads without thrashing React. 0..1 range. */
|
| 32 |
+
audioLevelRef: React.MutableRefObject<number>
|
| 33 |
+
/** Only fire barge-ins when this is true. Wire this to
|
| 34 |
+
* ``tts.isSpeaking && session.streamingNegotiated``. */
|
| 35 |
+
enabled: boolean
|
| 36 |
+
/** Callback fired once per speech-start episode. Debounced via
|
| 37 |
+
* ``minRefractoryMs`` to avoid double-firing on sustained speech. */
|
| 38 |
+
onBargeIn: () => void
|
| 39 |
+
/** Audio level (0..1) above which the tap considers the user to
|
| 40 |
+
* be speaking. Default 0.08 β see header comment on TTS bleed. */
|
| 41 |
+
threshold?: number
|
| 42 |
+
/** How long the level must stay above ``threshold`` before we
|
| 43 |
+
* declare speech-start. 80 ms matches the minimum voiced-phoneme
|
| 44 |
+
* duration, so a single keyboard clack doesn't trigger. */
|
| 45 |
+
minSustainMs?: number
|
| 46 |
+
/** After firing, ignore further triggers for this long. Prevents
|
| 47 |
+
* a single barge-in from firing twice while speech continues. */
|
| 48 |
+
minRefractoryMs?: number
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
/**
|
| 52 |
+
* Mount a VAD tap that fires ``onBargeIn`` when sustained speech is
|
| 53 |
+
* detected during an active TTS phase. Cleanup on unmount.
|
| 54 |
+
*/
|
| 55 |
+
export function useBargeInDetector({
|
| 56 |
+
audioLevelRef,
|
| 57 |
+
enabled,
|
| 58 |
+
onBargeIn,
|
| 59 |
+
threshold = 0.08,
|
| 60 |
+
minSustainMs = 80,
|
| 61 |
+
minRefractoryMs = 400,
|
| 62 |
+
}: UseBargeInDetectorArgs): void {
|
| 63 |
+
// Latest callback in a ref so a change in ``onBargeIn`` identity
|
| 64 |
+
// doesn't tear down the whole rAF loop.
|
| 65 |
+
const onBargeInRef = useRef(onBargeIn)
|
| 66 |
+
useEffect(() => {
|
| 67 |
+
onBargeInRef.current = onBargeIn
|
| 68 |
+
}, [onBargeIn])
|
| 69 |
+
|
| 70 |
+
useEffect(() => {
|
| 71 |
+
if (!enabled) return
|
| 72 |
+
let raf = 0
|
| 73 |
+
let aboveSince: number | null = null
|
| 74 |
+
let refractoryUntil = 0
|
| 75 |
+
const tick = () => {
|
| 76 |
+
const now = performance.now()
|
| 77 |
+
const level = audioLevelRef.current ?? 0
|
| 78 |
+
if (level >= threshold) {
|
| 79 |
+
if (aboveSince === null) aboveSince = now
|
| 80 |
+
else if (
|
| 81 |
+
now >= refractoryUntil &&
|
| 82 |
+
now - aboveSince >= minSustainMs
|
| 83 |
+
) {
|
| 84 |
+
refractoryUntil = now + minRefractoryMs
|
| 85 |
+
aboveSince = null
|
| 86 |
+
try {
|
| 87 |
+
onBargeInRef.current()
|
| 88 |
+
} catch {
|
| 89 |
+
/* listener errors must not crash the rAF loop */
|
| 90 |
+
}
|
| 91 |
+
}
|
| 92 |
+
} else {
|
| 93 |
+
aboveSince = null
|
| 94 |
+
}
|
| 95 |
+
raf = requestAnimationFrame(tick)
|
| 96 |
+
}
|
| 97 |
+
raf = requestAnimationFrame(tick)
|
| 98 |
+
return () => {
|
| 99 |
+
cancelAnimationFrame(raf)
|
| 100 |
+
}
|
| 101 |
+
}, [enabled, audioLevelRef, threshold, minSustainMs, minRefractoryMs])
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
// Exported for tests β pure detector without the React hook.
|
| 105 |
+
export function detectBargeIn(
|
| 106 |
+
samples: Array<{ tMs: number; level: number }>,
|
| 107 |
+
opts: {
|
| 108 |
+
threshold: number
|
| 109 |
+
minSustainMs: number
|
| 110 |
+
minRefractoryMs: number
|
| 111 |
+
},
|
| 112 |
+
): number[] {
|
| 113 |
+
const events: number[] = []
|
| 114 |
+
let aboveSince: number | null = null
|
| 115 |
+
let refractoryUntil = 0
|
| 116 |
+
for (const { tMs, level } of samples) {
|
| 117 |
+
if (level >= opts.threshold) {
|
| 118 |
+
if (aboveSince === null) aboveSince = tMs
|
| 119 |
+
else if (
|
| 120 |
+
tMs >= refractoryUntil &&
|
| 121 |
+
tMs - aboveSince >= opts.minSustainMs
|
| 122 |
+
) {
|
| 123 |
+
refractoryUntil = tMs + opts.minRefractoryMs
|
| 124 |
+
aboveSince = null
|
| 125 |
+
events.push(tMs)
|
| 126 |
+
}
|
| 127 |
+
} else {
|
| 128 |
+
aboveSince = null
|
| 129 |
+
}
|
| 130 |
+
}
|
| 131 |
+
return events
|
| 132 |
+
}
|
frontend/src/ui/call/callApi.ts
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* callApi.ts β REST wrapper for the voice_call session endpoint.
|
| 3 |
+
*
|
| 4 |
+
* Thin on purpose. Two responsibilities:
|
| 5 |
+
* 1. POST /v1/voice-call/sessions with the right shape
|
| 6 |
+
* 2. Normalize "feature unavailable" (404/501) vs hard errors
|
| 7 |
+
* so the hook can gracefully fall back to the chat-REST path
|
| 8 |
+
* when the backend has VOICE_CALL_ENABLED=false.
|
| 9 |
+
*
|
| 10 |
+
* Anything richer (envelope codec, seq bookkeeping, heartbeat,
|
| 11 |
+
* reconnect) lives in callSocket.ts β keeping those concerns out of
|
| 12 |
+
* this file is deliberate; the REST handshake runs once per call,
|
| 13 |
+
* the socket lifecycle runs for every envelope.
|
| 14 |
+
*/
|
| 15 |
+
|
| 16 |
+
import { resolveBackendUrl } from '../lib/backendUrl'
|
| 17 |
+
|
| 18 |
+
export interface CreateCallSessionRequest {
|
| 19 |
+
/** Existing chat conversation id to link this call to. Optional.
|
| 20 |
+
* The backend will create an ephemeral conversation if omitted. */
|
| 21 |
+
conversation_id?: string | null
|
| 22 |
+
/** Persona to carry into the turn. Null = assistant default. */
|
| 23 |
+
persona_id?: string | null
|
| 24 |
+
/** How the call was initiated. Matches backend CreateSessionReq. */
|
| 25 |
+
entry_mode?: 'voice' | 'call' | 'handoff'
|
| 26 |
+
/** Free-form client fingerprint (tz, platform, ua family).
|
| 27 |
+
* Also carries Phase 2/3 capability declarations:
|
| 28 |
+
* - ``streaming`` client can consume assistant.partial
|
| 29 |
+
* - ``barge_in`` client can detect user speech-start during
|
| 30 |
+
* TTS playback and send ``user.barge_in``
|
| 31 |
+
* The effective streaming mode for the session is
|
| 32 |
+
* ``min(client_declared, server_advertised)`` β both ends must
|
| 33 |
+
* agree or the turn runs in unary mode. Mismatches degrade
|
| 34 |
+
* gracefully without a hard error. */
|
| 35 |
+
device_info?: Record<string, unknown>
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
/** Server-advertised capabilities. Fields are optional so older
|
| 39 |
+
* servers (which don't emit them) are parsed safely and treated
|
| 40 |
+
* as "no". Used by useCallSession to decide whether to subscribe
|
| 41 |
+
* to the Phase 2 streaming events. */
|
| 42 |
+
export interface CallCapabilities {
|
| 43 |
+
/** Server supports assistant.partial / assistant.turn_end /
|
| 44 |
+
* assistant.cancel envelopes. */
|
| 45 |
+
streaming?: boolean
|
| 46 |
+
/** Server honours user.barge_in / transcript.partial as cancel
|
| 47 |
+
* signals. Only meaningful when ``streaming`` is also true. */
|
| 48 |
+
barge_in?: boolean
|
| 49 |
+
/** Legacy fields (pre-streaming). Kept in the interface so
|
| 50 |
+
* existing consumers don't need to widen the type. */
|
| 51 |
+
interruptions?: boolean
|
| 52 |
+
transcript_live?: boolean
|
| 53 |
+
[k: string]: unknown
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
export interface CreateCallSessionResponse {
|
| 57 |
+
session_id: string
|
| 58 |
+
/** Absolute or root-relative WebSocket URL to connect to. */
|
| 59 |
+
ws_url: string
|
| 60 |
+
/** Short-lived token the WS handshake must echo in its query. */
|
| 61 |
+
resume_token: string
|
| 62 |
+
/** When the session expires if never connected (ISO or epoch ms). */
|
| 63 |
+
expires_at: string | number
|
| 64 |
+
/** Hard cap enforced by the server. */
|
| 65 |
+
max_duration_sec: number
|
| 66 |
+
/** Backend capability advertisement (Β§ 3.1 of the streaming design). */
|
| 67 |
+
capabilities?: CallCapabilities
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
/** Error raised for any non-2xx from /v1/voice-call/sessions. The
|
| 71 |
+
* `isUnavailable` flag distinguishes "backend hasn't enabled this
|
| 72 |
+
* feature yet" from real failures the UI should surface. */
|
| 73 |
+
export class CallApiError extends Error {
|
| 74 |
+
constructor(
|
| 75 |
+
public readonly status: number,
|
| 76 |
+
public readonly bodyText: string,
|
| 77 |
+
) {
|
| 78 |
+
super(`voice-call session api ${status}${bodyText ? `: ${bodyText}` : ''}`)
|
| 79 |
+
this.name = 'CallApiError'
|
| 80 |
+
}
|
| 81 |
+
/** 404 = route not mounted (VOICE_CALL_ENABLED=false).
|
| 82 |
+
* 501 = feature recognized but disabled at runtime.
|
| 83 |
+
* The hook treats both as "fall back to chat REST". */
|
| 84 |
+
get isUnavailable(): boolean {
|
| 85 |
+
return this.status === 404 || this.status === 501
|
| 86 |
+
}
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
export async function createCallSession(
|
| 90 |
+
backendUrl: string,
|
| 91 |
+
req: CreateCallSessionRequest,
|
| 92 |
+
authToken?: string | null,
|
| 93 |
+
): Promise<CreateCallSessionResponse> {
|
| 94 |
+
// Defensive resolution: when the caller passes an empty or
|
| 95 |
+
// whitespace-only backend URL (e.g. a stale settings load), a
|
| 96 |
+
// naive ``replace(/\/+$/, '')`` leaves us with ``''`` and fetch
|
| 97 |
+
// falls back to the current page origin. On the Vite dev server
|
| 98 |
+
// (:3000) that silently routes to the SPA, which returns
|
| 99 |
+
// index.html and the subsequent JSON parse fails; in prod it can
|
| 100 |
+
// hit the wrong origin entirely. ``resolveBackendUrl`` ignores
|
| 101 |
+
// empty overrides and falls through to the app-wide resolver
|
| 102 |
+
// (localStorage β VITE_API_URL β origin β ``http://localhost:8000``),
|
| 103 |
+
// so we never accidentally issue a relative request.
|
| 104 |
+
const base = resolveBackendUrl(backendUrl)
|
| 105 |
+
const res = await fetch(`${base}/v1/voice-call/sessions`, {
|
| 106 |
+
method: 'POST',
|
| 107 |
+
headers: {
|
| 108 |
+
'content-type': 'application/json',
|
| 109 |
+
...(authToken ? { authorization: `Bearer ${authToken}` } : {}),
|
| 110 |
+
},
|
| 111 |
+
body: JSON.stringify({
|
| 112 |
+
conversation_id: req.conversation_id ?? null,
|
| 113 |
+
persona_id: req.persona_id ?? null,
|
| 114 |
+
entry_mode: req.entry_mode ?? 'call',
|
| 115 |
+
device_info: req.device_info ?? {},
|
| 116 |
+
}),
|
| 117 |
+
credentials: 'include',
|
| 118 |
+
})
|
| 119 |
+
|
| 120 |
+
if (!res.ok) {
|
| 121 |
+
const bodyText = await res.text().catch(() => '')
|
| 122 |
+
throw new CallApiError(res.status, bodyText)
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
const json = (await res.json()) as CreateCallSessionResponse
|
| 126 |
+
if (!json?.session_id || !json?.resume_token || !json?.ws_url) {
|
| 127 |
+
throw new CallApiError(res.status, 'malformed session payload')
|
| 128 |
+
}
|
| 129 |
+
return json
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
/** Declare the client's Phase 2/3 capabilities into the
|
| 133 |
+
* ``device_info`` blob. Set both to true in build configurations
|
| 134 |
+
* that include streamTts + bargeIn modules; set to false to run
|
| 135 |
+
* the session in forced-unary mode even against a streaming server
|
| 136 |
+
* (useful for bisecting regressions). */
|
| 137 |
+
export function clientStreamingCapabilities(
|
| 138 |
+
base: Record<string, unknown>,
|
| 139 |
+
enabled: boolean,
|
| 140 |
+
): Record<string, unknown> {
|
| 141 |
+
if (!enabled) return base
|
| 142 |
+
return { ...base, streaming: true, barge_in: true }
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
/** Compute the effective streaming mode from the server's capability
|
| 146 |
+
* response and the client's declared support. Truth table in Β§ 3.1
|
| 147 |
+
* of docs/analysis/voice-call-streaming-design.md. */
|
| 148 |
+
export function effectiveStreamingMode(
|
| 149 |
+
serverCaps: CallCapabilities | undefined,
|
| 150 |
+
clientStreaming: boolean,
|
| 151 |
+
): { streaming: boolean; barge_in: boolean } {
|
| 152 |
+
const streaming = !!(serverCaps?.streaming && clientStreaming)
|
| 153 |
+
const barge_in = streaming && !!serverCaps?.barge_in
|
| 154 |
+
return { streaming, barge_in }
|
| 155 |
+
}
|
| 156 |
+
|
| 157 |
+
/** Resolve a potentially-relative ws_url against the current page
|
| 158 |
+
* and the backend URL. Backends sometimes return `/v1/voice-call/ws`
|
| 159 |
+
* (path only) so the client can pick the right scheme + host. */
|
| 160 |
+
export function resolveWsUrl(
|
| 161 |
+
wsUrl: string,
|
| 162 |
+
sessionId: string,
|
| 163 |
+
resumeToken: string,
|
| 164 |
+
authToken: string | null,
|
| 165 |
+
backendUrl: string,
|
| 166 |
+
): string {
|
| 167 |
+
let url: URL
|
| 168 |
+
if (/^wss?:\/\//i.test(wsUrl)) {
|
| 169 |
+
url = new URL(wsUrl)
|
| 170 |
+
} else if (wsUrl.startsWith('//')) {
|
| 171 |
+
url = new URL(`${location.protocol === 'https:' ? 'wss:' : 'ws:'}${wsUrl}`)
|
| 172 |
+
} else if (wsUrl.startsWith('/')) {
|
| 173 |
+
// Root-relative β derive scheme + host from backendUrl.
|
| 174 |
+
const backend = new URL(backendUrl)
|
| 175 |
+
const scheme = backend.protocol === 'https:' ? 'wss:' : 'ws:'
|
| 176 |
+
url = new URL(`${scheme}//${backend.host}${wsUrl}`)
|
| 177 |
+
} else {
|
| 178 |
+
url = new URL(wsUrl, backendUrl)
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
// Inject session_id if the server gave us a template (rare) or
|
| 182 |
+
// appended nothing; the standard shape is
|
| 183 |
+
// /v1/voice-call/ws/{session_id}.
|
| 184 |
+
if (!url.pathname.endsWith(sessionId)) {
|
| 185 |
+
url.pathname = url.pathname.replace(/\/+$/, '') + `/${sessionId}`
|
| 186 |
+
}
|
| 187 |
+
url.searchParams.set('resume_token', resumeToken)
|
| 188 |
+
if (authToken) url.searchParams.set('token', authToken)
|
| 189 |
+
return url.toString()
|
| 190 |
+
}
|
frontend/src/ui/call/callSocket.ts
ADDED
|
@@ -0,0 +1,579 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* callSocket.ts β typed transport for the voice_call WebSocket.
|
| 3 |
+
*
|
| 4 |
+
* Owns the connection, the envelope codec, monotonic sequence
|
| 5 |
+
* tracking, the heartbeat, graceful shutdown, and reconnect-with-
|
| 6 |
+
* backoff semantics. Exposes a narrow event surface that the React
|
| 7 |
+
* hook (useCallSession) binds to; callers never touch the raw socket.
|
| 8 |
+
*
|
| 9 |
+
* Explicit state machine β the only transitions allowed are:
|
| 10 |
+
*
|
| 11 |
+
* idle ββconnect()βββΆ connecting ββopenββββββββΆ live
|
| 12 |
+
* β β
|
| 13 |
+
* β (handshake failure) β (transient drop w/ resume window)
|
| 14 |
+
* βΌ βΌ
|
| 15 |
+
* closed reconnecting ββ(give up / expired)βββΆ closed
|
| 16 |
+
* β
|
| 17 |
+
* ββββββ(socket open)ββββββββΆ live
|
| 18 |
+
*
|
| 19 |
+
* live ββclose('user_ended' | 'max_duration' | 'idle')ββββΆ closed
|
| 20 |
+
*
|
| 21 |
+
* Industry choices baked in:
|
| 22 |
+
* β’ Exponential backoff with full jitter (AWS / Cloudflare canonical
|
| 23 |
+
* implementation) β stops thundering herds on a shared-backend
|
| 24 |
+
* brown-out.
|
| 25 |
+
* β’ Heartbeat treats silence as liveness failure β we send a
|
| 26 |
+
* client-originated `call.control ping` every N seconds AND
|
| 27 |
+
* watch for server-side `ping` events; if BOTH are silent for
|
| 28 |
+
* 2Γ the interval, the connection is considered dead and the
|
| 29 |
+
* backoff loop fires.
|
| 30 |
+
* β’ Outbound envelopes are queued while reconnecting. When the
|
| 31 |
+
* socket reopens within the resume window the queue is flushed
|
| 32 |
+
* in-order; otherwise it's dropped on shutdown.
|
| 33 |
+
* β’ Resume semantics honoured: one reconnect attempt carries the
|
| 34 |
+
* same session_id + resume_token; if the server returns
|
| 35 |
+
* 1008 resume-expired we give up and surface 'resume_expired'.
|
| 36 |
+
* β’ Every event emitted by this module is typed; no `any`, no
|
| 37 |
+
* ad-hoc payload parsing inside listeners.
|
| 38 |
+
*/
|
| 39 |
+
|
| 40 |
+
// ββ Typed envelope contracts βββββββββββββββββββββββββββββββββββββββ
|
| 41 |
+
// Mirror the backend ws.py contract. Keep in sync with
|
| 42 |
+
// docs/analysis/voice-call-human-simulation-design.md Β§ 6.
|
| 43 |
+
|
| 44 |
+
export type CallLifecycleStatus =
|
| 45 |
+
| 'idle'
|
| 46 |
+
| 'connecting'
|
| 47 |
+
| 'live'
|
| 48 |
+
| 'reconnecting'
|
| 49 |
+
| 'draining'
|
| 50 |
+
| 'closed'
|
| 51 |
+
|
| 52 |
+
export type CallCloseReason =
|
| 53 |
+
| 'user_ended'
|
| 54 |
+
| 'max_duration'
|
| 55 |
+
| 'idle_timeout'
|
| 56 |
+
| 'resume_expired'
|
| 57 |
+
| 'bad_resume_token'
|
| 58 |
+
| 'session_not_found'
|
| 59 |
+
| 'session_ended'
|
| 60 |
+
| 'websocket_disabled'
|
| 61 |
+
| 'network_error'
|
| 62 |
+
| 'unmounted'
|
| 63 |
+
| 'unknown'
|
| 64 |
+
|
| 65 |
+
export interface ServerEnvelope<
|
| 66 |
+
T extends string = string,
|
| 67 |
+
P extends Record<string, unknown> = Record<string, unknown>,
|
| 68 |
+
> {
|
| 69 |
+
type: T
|
| 70 |
+
seq: number
|
| 71 |
+
ts: number
|
| 72 |
+
payload: P
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
// Server β client event payloads. One type per server `type`; if
|
| 76 |
+
// the backend ever adds a new event, it lands in `unknown` and is
|
| 77 |
+
// dropped β not a runtime crash.
|
| 78 |
+
|
| 79 |
+
export interface CallStatePayload {
|
| 80 |
+
status: 'live' | 'ending' | 'ended'
|
| 81 |
+
reason?: 'user_ended' | 'idle' | 'max_duration'
|
| 82 |
+
since?: number
|
| 83 |
+
}
|
| 84 |
+
export interface AssistantTranscriptPayload {
|
| 85 |
+
role: 'assistant'
|
| 86 |
+
text: string
|
| 87 |
+
}
|
| 88 |
+
export interface AssistantFillerPayload {
|
| 89 |
+
token: string
|
| 90 |
+
ts?: number
|
| 91 |
+
session_id?: string
|
| 92 |
+
}
|
| 93 |
+
export interface AssistantBackchannelPayload {
|
| 94 |
+
token: string
|
| 95 |
+
volume_db?: number
|
| 96 |
+
ts?: number
|
| 97 |
+
}
|
| 98 |
+
export interface ServerErrorPayload {
|
| 99 |
+
code: string
|
| 100 |
+
message: string
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
// Phase 2/3 payloads (Β§ 3.2 of the streaming design doc).
|
| 104 |
+
|
| 105 |
+
export interface AssistantPartialPayload {
|
| 106 |
+
turn_id: string
|
| 107 |
+
/** The NEW text only. Callers concatenate deltas per turn_id. */
|
| 108 |
+
delta: string
|
| 109 |
+
/** Monotonic per-turn index starting at 0. Advisory (the WS
|
| 110 |
+
* already enforces monotonic seq); useful for replay logs. */
|
| 111 |
+
index: number
|
| 112 |
+
}
|
| 113 |
+
export interface AssistantTurnEndPayload {
|
| 114 |
+
turn_id: string
|
| 115 |
+
reason: 'complete' | 'cancelled' | 'error'
|
| 116 |
+
/** Authoritative concatenation of this turn's deltas (server-side).
|
| 117 |
+
* Clients can use this directly instead of maintaining their own
|
| 118 |
+
* buffer. Optional in principle β but our server always emits it. */
|
| 119 |
+
full_text?: string
|
| 120 |
+
}
|
| 121 |
+
export interface AssistantCancelPayload {
|
| 122 |
+
turn_id: string
|
| 123 |
+
cause?: 'user_barge_in' | 'user_partial' | string
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
// Client β server. Keep the types literal so the compiler catches
|
| 127 |
+
// typos at the send() call site.
|
| 128 |
+
|
| 129 |
+
export interface UiStatePayload {
|
| 130 |
+
muted?: boolean
|
| 131 |
+
speaker_on?: boolean
|
| 132 |
+
backgrounded?: boolean
|
| 133 |
+
}
|
| 134 |
+
export interface TranscriptFinalPayload {
|
| 135 |
+
text: string
|
| 136 |
+
model?: string
|
| 137 |
+
lang?: string
|
| 138 |
+
}
|
| 139 |
+
export type CallControlAction = 'end' | 'ping'
|
| 140 |
+
|
| 141 |
+
// ββ Typed event bus for callers ββββββββββββββββββββββββββββββββββββ
|
| 142 |
+
|
| 143 |
+
export interface CallSocketEventMap {
|
| 144 |
+
statusChange: CallLifecycleStatus
|
| 145 |
+
callState: CallStatePayload
|
| 146 |
+
/** Still fires for turns that ran in unary mode (flag off OR
|
| 147 |
+
* capability mismatch). Streaming turns use the three events
|
| 148 |
+
* below instead. */
|
| 149 |
+
assistantTranscript: AssistantTranscriptPayload
|
| 150 |
+
/** Phase 2: one delta per server emission. */
|
| 151 |
+
assistantPartial: AssistantPartialPayload
|
| 152 |
+
/** Phase 2: exactly once per streamed turn, with a final reason. */
|
| 153 |
+
assistantTurnEnd: AssistantTurnEndPayload
|
| 154 |
+
/** Phase 3: server acks a barge-in; client stops TTS on receipt. */
|
| 155 |
+
assistantCancel: AssistantCancelPayload
|
| 156 |
+
assistantFiller: AssistantFillerPayload
|
| 157 |
+
assistantBackchannel: AssistantBackchannelPayload
|
| 158 |
+
serverError: ServerErrorPayload
|
| 159 |
+
safetyNotice: Record<string, unknown>
|
| 160 |
+
pong: void
|
| 161 |
+
closed: { reason: CallCloseReason; code?: number; detail?: string }
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
type Listener<E extends keyof CallSocketEventMap> = (
|
| 165 |
+
payload: CallSocketEventMap[E],
|
| 166 |
+
) => void
|
| 167 |
+
|
| 168 |
+
// ββ Backoff helper (AWS-style full jitter) βββββββββββββββββββββββββ
|
| 169 |
+
|
| 170 |
+
function backoffMs(attempt: number, baseMs = 500, capMs = 10_000): number {
|
| 171 |
+
const exp = Math.min(capMs, baseMs * 2 ** attempt)
|
| 172 |
+
return Math.floor(Math.random() * exp)
|
| 173 |
+
}
|
| 174 |
+
|
| 175 |
+
// ββ Close-code β reason mapping ββββββββββββββββββββββββββββββββββββ
|
| 176 |
+
|
| 177 |
+
const POLICY_VIOLATION = 1008
|
| 178 |
+
const NORMAL_CLOSURE = 1000
|
| 179 |
+
|
| 180 |
+
function closeReasonFrom(code: number | undefined, text: string): CallCloseReason {
|
| 181 |
+
const reason = (text || '').toLowerCase()
|
| 182 |
+
if (code === POLICY_VIOLATION) {
|
| 183 |
+
if (reason.includes('resume-expired')) return 'resume_expired'
|
| 184 |
+
if (reason.includes('bad-resume-token')) return 'bad_resume_token'
|
| 185 |
+
if (reason.includes('session-not-found')) return 'session_not_found'
|
| 186 |
+
if (reason.includes('session-ended')) return 'session_ended'
|
| 187 |
+
if (reason.includes('websocket-disabled')) return 'websocket_disabled'
|
| 188 |
+
}
|
| 189 |
+
if (code === NORMAL_CLOSURE && reason.includes('max-duration')) return 'max_duration'
|
| 190 |
+
if (reason.includes('idle')) return 'idle_timeout'
|
| 191 |
+
return 'unknown'
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
// ββ Construction options βββββββββββββββββββββββββββββββββββββββββββ
|
| 195 |
+
|
| 196 |
+
export interface CallSocketOptions {
|
| 197 |
+
/** Fully-resolved WS URL including ?resume_token=β¦ and ?token=β¦. */
|
| 198 |
+
url: string
|
| 199 |
+
/** How long the backend will honour a resume. Used to decide when to
|
| 200 |
+
* give up reconnecting. Default 20 s (matches VOICE_CALL_RESUME_WINDOW_SEC). */
|
| 201 |
+
resumeWindowSec?: number
|
| 202 |
+
/** How often we send a client-originated ping. Default 15 s. */
|
| 203 |
+
heartbeatIntervalMs?: number
|
| 204 |
+
/** Dependency injection for tests β defaults to global WebSocket. */
|
| 205 |
+
webSocketImpl?: typeof WebSocket
|
| 206 |
+
/** Structured log sink. Defaults to console.{info,warn,error} with
|
| 207 |
+
* any `resume_token`, `token`, or `jwt` fields redacted by the
|
| 208 |
+
* caller. Keep string-typed; don't pass raw envelopes in. */
|
| 209 |
+
log?: {
|
| 210 |
+
info: (msg: string, extra?: Record<string, unknown>) => void
|
| 211 |
+
warn: (msg: string, extra?: Record<string, unknown>) => void
|
| 212 |
+
error: (msg: string, extra?: Record<string, unknown>) => void
|
| 213 |
+
}
|
| 214 |
+
}
|
| 215 |
+
|
| 216 |
+
// ββ CallSocket class βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 217 |
+
|
| 218 |
+
export class CallSocket {
|
| 219 |
+
private ws: WebSocket | null = null
|
| 220 |
+
private status: CallLifecycleStatus = 'idle'
|
| 221 |
+
private lastServerSeq = 0
|
| 222 |
+
private outboundQueue: string[] = []
|
| 223 |
+
private heartbeatTimer: number | null = null
|
| 224 |
+
private lastServerActivityMs = 0
|
| 225 |
+
private deadPeerTimer: number | null = null
|
| 226 |
+
private reconnectAttempt = 0
|
| 227 |
+
private reconnectTimer: number | null = null
|
| 228 |
+
private connectedAtMs = 0
|
| 229 |
+
private shuttingDown = false
|
| 230 |
+
|
| 231 |
+
private readonly listeners: {
|
| 232 |
+
[K in keyof CallSocketEventMap]?: Set<Listener<K>>
|
| 233 |
+
} = {}
|
| 234 |
+
|
| 235 |
+
private readonly url: string
|
| 236 |
+
private readonly resumeWindowMs: number
|
| 237 |
+
private readonly heartbeatIntervalMs: number
|
| 238 |
+
private readonly WebSocketImpl: typeof WebSocket
|
| 239 |
+
private readonly log: NonNullable<CallSocketOptions['log']>
|
| 240 |
+
|
| 241 |
+
constructor(opts: CallSocketOptions) {
|
| 242 |
+
this.url = opts.url
|
| 243 |
+
this.resumeWindowMs = (opts.resumeWindowSec ?? 20) * 1000
|
| 244 |
+
this.heartbeatIntervalMs = opts.heartbeatIntervalMs ?? 15_000
|
| 245 |
+
this.WebSocketImpl = opts.webSocketImpl ?? WebSocket
|
| 246 |
+
this.log = opts.log ?? {
|
| 247 |
+
info: (m, e) => console.info(`[callSocket] ${m}`, e ?? ''),
|
| 248 |
+
warn: (m, e) => console.warn(`[callSocket] ${m}`, e ?? ''),
|
| 249 |
+
error: (m, e) => console.error(`[callSocket] ${m}`, e ?? ''),
|
| 250 |
+
}
|
| 251 |
+
}
|
| 252 |
+
|
| 253 |
+
// Public API ----------------------------------------------------
|
| 254 |
+
|
| 255 |
+
getStatus(): CallLifecycleStatus { return this.status }
|
| 256 |
+
|
| 257 |
+
on<E extends keyof CallSocketEventMap>(evt: E, fn: Listener<E>): () => void {
|
| 258 |
+
let set = this.listeners[evt] as Set<Listener<E>> | undefined
|
| 259 |
+
if (!set) {
|
| 260 |
+
set = new Set<Listener<E>>()
|
| 261 |
+
// Narrow through `as unknown` β TS can't track the generic
|
| 262 |
+
// discriminant through the indexed property set.
|
| 263 |
+
;(this.listeners as Record<string, unknown>)[evt] = set
|
| 264 |
+
}
|
| 265 |
+
set.add(fn)
|
| 266 |
+
return () => set!.delete(fn)
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
/** Open the socket. Safe to call exactly once per instance. */
|
| 270 |
+
connect(): void {
|
| 271 |
+
if (this.status !== 'idle') {
|
| 272 |
+
this.log.warn('connect() called in non-idle state', { status: this.status })
|
| 273 |
+
return
|
| 274 |
+
}
|
| 275 |
+
this.openSocket()
|
| 276 |
+
}
|
| 277 |
+
|
| 278 |
+
/** Send the captured user transcript as a chat turn. Queued if the
|
| 279 |
+
* socket is currently mid-reconnect. */
|
| 280 |
+
sendTranscript(p: TranscriptFinalPayload): void {
|
| 281 |
+
this.enqueueLine({ type: 'transcript.final', ts: Date.now(), payload: p })
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
/** Bookkeeping β muted, speaker on/off, app backgrounded. */
|
| 285 |
+
sendUiState(p: UiStatePayload): void {
|
| 286 |
+
this.enqueueLine({ type: 'ui.state', ts: Date.now(), payload: p })
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
/** Control channel β 'ping' requests a pong; 'end' terminates. */
|
| 290 |
+
sendControl(action: CallControlAction): void {
|
| 291 |
+
this.enqueueLine({ type: 'call.control', ts: Date.now(), payload: { action } })
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
/** Phase 2 β interim STT output while the user is still speaking.
|
| 295 |
+
* Server uses it as a secondary barge-in trigger and, in a
|
| 296 |
+
* future phase, for semantic endpointing. Safe no-op against a
|
| 297 |
+
* non-streaming server (unknown type is dropped). */
|
| 298 |
+
sendTranscriptPartial(p: { text: string; stable_prefix_len?: number }): void {
|
| 299 |
+
this.enqueueLine({
|
| 300 |
+
type: 'transcript.partial',
|
| 301 |
+
ts: Date.now(),
|
| 302 |
+
payload: p,
|
| 303 |
+
})
|
| 304 |
+
}
|
| 305 |
+
|
| 306 |
+
/** Phase 3 β explicit interrupt. Fired the instant the client's
|
| 307 |
+
* VAD trips above threshold while an assistant turn is in-flight.
|
| 308 |
+
* ``turn_id`` disambiguates against a stale signal racing a new
|
| 309 |
+
* turn; the server compares ids and silently drops mismatches. */
|
| 310 |
+
sendBargeIn(turn_id: string): void {
|
| 311 |
+
this.enqueueLine({
|
| 312 |
+
type: 'user.barge_in',
|
| 313 |
+
ts: Date.now(),
|
| 314 |
+
payload: { turn_id },
|
| 315 |
+
})
|
| 316 |
+
}
|
| 317 |
+
|
| 318 |
+
/** Graceful user-initiated end. Sends control 'end', then closes. */
|
| 319 |
+
end(): void {
|
| 320 |
+
if (this.shuttingDown) return
|
| 321 |
+
this.shuttingDown = true
|
| 322 |
+
this.transition('draining')
|
| 323 |
+
this.sendControl('end')
|
| 324 |
+
// Give the server ~400 ms to echo call.state {status:'ended'}
|
| 325 |
+
// before we force-close; prevents the socket from looking
|
| 326 |
+
// orphaned in backend logs.
|
| 327 |
+
window.setTimeout(() => this.dispose('user_ended'), 400)
|
| 328 |
+
}
|
| 329 |
+
|
| 330 |
+
/** Teardown without a control 'end' β used when the component
|
| 331 |
+
* unmounts, the user navigates away, or a terminal error occurs. */
|
| 332 |
+
dispose(reason: CallCloseReason = 'unmounted', code = NORMAL_CLOSURE): void {
|
| 333 |
+
if (this.status === 'closed') return
|
| 334 |
+
this.shuttingDown = true
|
| 335 |
+
this.clearTimers()
|
| 336 |
+
try { this.ws?.close(code, reason) } catch { /* already closed */ }
|
| 337 |
+
this.ws = null
|
| 338 |
+
this.outboundQueue = []
|
| 339 |
+
this.transition('closed')
|
| 340 |
+
this.emit('closed', { reason, code })
|
| 341 |
+
}
|
| 342 |
+
|
| 343 |
+
// Socket lifecycle ----------------------------------------------
|
| 344 |
+
|
| 345 |
+
private openSocket(): void {
|
| 346 |
+
this.transition(this.reconnectAttempt === 0 ? 'connecting' : 'reconnecting')
|
| 347 |
+
let ws: WebSocket
|
| 348 |
+
try {
|
| 349 |
+
ws = new this.WebSocketImpl(this.url)
|
| 350 |
+
} catch (err) {
|
| 351 |
+
this.log.error('WebSocket constructor threw', { err: String(err) })
|
| 352 |
+
this.scheduleReconnect()
|
| 353 |
+
return
|
| 354 |
+
}
|
| 355 |
+
this.ws = ws
|
| 356 |
+
this.lastServerActivityMs = Date.now()
|
| 357 |
+
|
| 358 |
+
ws.addEventListener('open', () => {
|
| 359 |
+
this.reconnectAttempt = 0
|
| 360 |
+
this.connectedAtMs = Date.now()
|
| 361 |
+
this.transition('live')
|
| 362 |
+
this.startHeartbeat()
|
| 363 |
+
this.flushQueue()
|
| 364 |
+
})
|
| 365 |
+
|
| 366 |
+
ws.addEventListener('message', (ev) => this.onMessage(ev))
|
| 367 |
+
|
| 368 |
+
ws.addEventListener('close', (ev) => this.onClose(ev))
|
| 369 |
+
|
| 370 |
+
ws.addEventListener('error', () => {
|
| 371 |
+
// The browser fires `error` right before `close` on network
|
| 372 |
+
// failures. We don't transition here β onClose() decides
|
| 373 |
+
// reconnect vs terminal based on the code.
|
| 374 |
+
this.log.warn('socket error event')
|
| 375 |
+
})
|
| 376 |
+
}
|
| 377 |
+
|
| 378 |
+
private onMessage(ev: MessageEvent): void {
|
| 379 |
+
this.lastServerActivityMs = Date.now()
|
| 380 |
+
if (typeof ev.data !== 'string') return
|
| 381 |
+
|
| 382 |
+
let env: ServerEnvelope | null = null
|
| 383 |
+
try {
|
| 384 |
+
env = JSON.parse(ev.data) as ServerEnvelope
|
| 385 |
+
} catch {
|
| 386 |
+
this.log.warn('non-JSON frame dropped')
|
| 387 |
+
return
|
| 388 |
+
}
|
| 389 |
+
if (!env || typeof env.type !== 'string') return
|
| 390 |
+
|
| 391 |
+
// Monotonic seq validation. If the server ever ships out-of-order
|
| 392 |
+
// frames we log once and keep going β correctness trumps strictness.
|
| 393 |
+
if (typeof env.seq === 'number' && env.seq <= this.lastServerSeq) {
|
| 394 |
+
this.log.warn('non-monotonic seq', {
|
| 395 |
+
got: env.seq, last: this.lastServerSeq, type: env.type,
|
| 396 |
+
})
|
| 397 |
+
} else if (typeof env.seq === 'number') {
|
| 398 |
+
this.lastServerSeq = env.seq
|
| 399 |
+
}
|
| 400 |
+
|
| 401 |
+
this.dispatch(env)
|
| 402 |
+
}
|
| 403 |
+
|
| 404 |
+
private dispatch(env: ServerEnvelope): void {
|
| 405 |
+
const raw = env.payload as unknown
|
| 406 |
+
switch (env.type) {
|
| 407 |
+
case 'call.state': {
|
| 408 |
+
const p = raw as CallStatePayload
|
| 409 |
+
this.emit('callState', p)
|
| 410 |
+
if (p.status === 'ended') this.dispose('user_ended')
|
| 411 |
+
return
|
| 412 |
+
}
|
| 413 |
+
case 'transcript.final':
|
| 414 |
+
this.emit('assistantTranscript', raw as AssistantTranscriptPayload)
|
| 415 |
+
return
|
| 416 |
+
case 'assistant.partial':
|
| 417 |
+
this.emit('assistantPartial', raw as AssistantPartialPayload)
|
| 418 |
+
return
|
| 419 |
+
case 'assistant.turn_end':
|
| 420 |
+
this.emit('assistantTurnEnd', raw as AssistantTurnEndPayload)
|
| 421 |
+
return
|
| 422 |
+
case 'assistant.cancel':
|
| 423 |
+
this.emit('assistantCancel', raw as AssistantCancelPayload)
|
| 424 |
+
return
|
| 425 |
+
case 'assistant.filler':
|
| 426 |
+
this.emit('assistantFiller', raw as AssistantFillerPayload)
|
| 427 |
+
return
|
| 428 |
+
case 'assistant.backchannel':
|
| 429 |
+
this.emit('assistantBackchannel', raw as AssistantBackchannelPayload)
|
| 430 |
+
return
|
| 431 |
+
case 'safety.notice':
|
| 432 |
+
this.emit('safetyNotice', env.payload)
|
| 433 |
+
return
|
| 434 |
+
case 'error':
|
| 435 |
+
this.emit('serverError', raw as ServerErrorPayload)
|
| 436 |
+
return
|
| 437 |
+
case 'pong':
|
| 438 |
+
this.emit('pong', undefined as void)
|
| 439 |
+
return
|
| 440 |
+
case 'ping':
|
| 441 |
+
// Server heartbeat β immediately echo to keep the liveness
|
| 442 |
+
// counters tight on both ends.
|
| 443 |
+
this.sendControl('ping')
|
| 444 |
+
return
|
| 445 |
+
default:
|
| 446 |
+
// Forward-compat: unknown event types are no-ops.
|
| 447 |
+
this.log.info(`unknown server event: ${env.type}`)
|
| 448 |
+
}
|
| 449 |
+
}
|
| 450 |
+
|
| 451 |
+
private onClose(ev: CloseEvent): void {
|
| 452 |
+
const reason = closeReasonFrom(ev.code, ev.reason)
|
| 453 |
+
this.clearTimers()
|
| 454 |
+
|
| 455 |
+
// Terminal close codes: don't reconnect.
|
| 456 |
+
const terminal: ReadonlySet<CallCloseReason> = new Set([
|
| 457 |
+
'user_ended',
|
| 458 |
+
'max_duration',
|
| 459 |
+
'resume_expired',
|
| 460 |
+
'bad_resume_token',
|
| 461 |
+
'session_not_found',
|
| 462 |
+
'session_ended',
|
| 463 |
+
'websocket_disabled',
|
| 464 |
+
])
|
| 465 |
+
if (this.shuttingDown || terminal.has(reason)) {
|
| 466 |
+
this.ws = null
|
| 467 |
+
this.transition('closed')
|
| 468 |
+
this.emit('closed', { reason, code: ev.code, detail: ev.reason })
|
| 469 |
+
return
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
// Transient β try to reconnect within the resume window.
|
| 473 |
+
this.log.warn('socket dropped; scheduling reconnect', {
|
| 474 |
+
code: ev.code, reason: ev.reason,
|
| 475 |
+
})
|
| 476 |
+
this.ws = null
|
| 477 |
+
this.scheduleReconnect()
|
| 478 |
+
}
|
| 479 |
+
|
| 480 |
+
private scheduleReconnect(): void {
|
| 481 |
+
if (this.shuttingDown) return
|
| 482 |
+
const sinceLiveMs = this.connectedAtMs > 0 ? Date.now() - this.connectedAtMs : 0
|
| 483 |
+
if (sinceLiveMs > this.resumeWindowMs) {
|
| 484 |
+
this.log.warn('resume window elapsed; giving up')
|
| 485 |
+
this.dispose('resume_expired', POLICY_VIOLATION)
|
| 486 |
+
return
|
| 487 |
+
}
|
| 488 |
+
const delay = backoffMs(this.reconnectAttempt)
|
| 489 |
+
this.reconnectAttempt += 1
|
| 490 |
+
this.transition('reconnecting')
|
| 491 |
+
this.reconnectTimer = window.setTimeout(() => {
|
| 492 |
+
this.reconnectTimer = null
|
| 493 |
+
this.openSocket()
|
| 494 |
+
}, delay)
|
| 495 |
+
}
|
| 496 |
+
|
| 497 |
+
// Queue + heartbeat ---------------------------------------------
|
| 498 |
+
|
| 499 |
+
private enqueueLine(env: { type: string; ts: number; payload: unknown }): void {
|
| 500 |
+
const line = JSON.stringify(env)
|
| 501 |
+
if (this.status === 'live' && this.ws?.readyState === this.WebSocketImpl.OPEN) {
|
| 502 |
+
try { this.ws.send(line) } catch (err) {
|
| 503 |
+
this.log.warn('send failed; queueing', { err: String(err) })
|
| 504 |
+
this.outboundQueue.push(line)
|
| 505 |
+
}
|
| 506 |
+
} else {
|
| 507 |
+
this.outboundQueue.push(line)
|
| 508 |
+
}
|
| 509 |
+
}
|
| 510 |
+
|
| 511 |
+
private flushQueue(): void {
|
| 512 |
+
if (!this.ws || this.ws.readyState !== this.WebSocketImpl.OPEN) return
|
| 513 |
+
const q = this.outboundQueue
|
| 514 |
+
this.outboundQueue = []
|
| 515 |
+
for (const line of q) {
|
| 516 |
+
try { this.ws.send(line) } catch (err) {
|
| 517 |
+
this.log.error('flush failed; re-queueing remaining', { err: String(err) })
|
| 518 |
+
this.outboundQueue.push(line)
|
| 519 |
+
return
|
| 520 |
+
}
|
| 521 |
+
}
|
| 522 |
+
}
|
| 523 |
+
|
| 524 |
+
private startHeartbeat(): void {
|
| 525 |
+
this.clearHeartbeat()
|
| 526 |
+
this.heartbeatTimer = window.setInterval(() => {
|
| 527 |
+
this.sendControl('ping')
|
| 528 |
+
}, this.heartbeatIntervalMs)
|
| 529 |
+
|
| 530 |
+
// Dead-peer watchdog β if we haven't heard from the server in
|
| 531 |
+
// 2Γ the heartbeat interval, force a reconnect. Avoids hanging
|
| 532 |
+
// forever on half-open TCP sockets (mobile NAT rebinds).
|
| 533 |
+
this.deadPeerTimer = window.setInterval(() => {
|
| 534 |
+
if (Date.now() - this.lastServerActivityMs > this.heartbeatIntervalMs * 2) {
|
| 535 |
+
this.log.warn('peer silent; forcing reconnect')
|
| 536 |
+
try { this.ws?.close() } catch { /* ignore */ }
|
| 537 |
+
}
|
| 538 |
+
}, this.heartbeatIntervalMs)
|
| 539 |
+
}
|
| 540 |
+
|
| 541 |
+
private clearHeartbeat(): void {
|
| 542 |
+
if (this.heartbeatTimer !== null) {
|
| 543 |
+
window.clearInterval(this.heartbeatTimer)
|
| 544 |
+
this.heartbeatTimer = null
|
| 545 |
+
}
|
| 546 |
+
if (this.deadPeerTimer !== null) {
|
| 547 |
+
window.clearInterval(this.deadPeerTimer)
|
| 548 |
+
this.deadPeerTimer = null
|
| 549 |
+
}
|
| 550 |
+
}
|
| 551 |
+
|
| 552 |
+
private clearTimers(): void {
|
| 553 |
+
this.clearHeartbeat()
|
| 554 |
+
if (this.reconnectTimer !== null) {
|
| 555 |
+
window.clearTimeout(this.reconnectTimer)
|
| 556 |
+
this.reconnectTimer = null
|
| 557 |
+
}
|
| 558 |
+
}
|
| 559 |
+
|
| 560 |
+
// Emit + state transition ---------------------------------------
|
| 561 |
+
|
| 562 |
+
private emit<E extends keyof CallSocketEventMap>(
|
| 563 |
+
evt: E, payload: CallSocketEventMap[E],
|
| 564 |
+
): void {
|
| 565 |
+
const set = this.listeners[evt] as Set<Listener<E>> | undefined
|
| 566 |
+
if (!set) return
|
| 567 |
+
for (const fn of set) {
|
| 568 |
+
try { fn(payload) } catch (err) {
|
| 569 |
+
this.log.error('listener threw', { evt, err: String(err) })
|
| 570 |
+
}
|
| 571 |
+
}
|
| 572 |
+
}
|
| 573 |
+
|
| 574 |
+
private transition(next: CallLifecycleStatus): void {
|
| 575 |
+
if (this.status === next) return
|
| 576 |
+
this.status = next
|
| 577 |
+
this.emit('statusChange', next)
|
| 578 |
+
}
|
| 579 |
+
}
|
frontend/src/ui/call/log.ts
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Structured call-lifecycle logger + shared TTS ownership.
|
| 3 |
+
*
|
| 4 |
+
* Three pieces, shipped together because they're tightly coupled:
|
| 5 |
+
*
|
| 6 |
+
* clog(evt) Namespaced, grep-friendly console log
|
| 7 |
+
* for every call-related event. Format
|
| 8 |
+
* ``[Call+<ms>] {e:β¦, β¦}`` so
|
| 9 |
+
* ``grep "\\[Call\\]"`` surfaces a
|
| 10 |
+
* chronological trace of any session.
|
| 11 |
+
*
|
| 12 |
+
* speakOwned(source, text) Single-owner TTS dispatch. Both the
|
| 13 |
+
* overlay's fallback opener and the
|
| 14 |
+
* App-level assistant-reply effect route
|
| 15 |
+
* through this function, which guards
|
| 16 |
+
* on ``window.__hp_tts_owner__`` so
|
| 17 |
+
* duplicate sources don't double-speak
|
| 18 |
+
* the same turn. First to call wins;
|
| 19 |
+
* the second sees ``owner !== source``
|
| 20 |
+
* and drops cleanly (emits an
|
| 21 |
+
* ``tts: dedupe`` log so the drop is
|
| 22 |
+
* auditable).
|
| 23 |
+
*
|
| 24 |
+
* isCallFullDuplexEnabled() Feature flag gate. Defaults to ON; a
|
| 25 |
+
* user can force-off via
|
| 26 |
+
* ``localStorage.homepilot_call_full_duplex``
|
| 27 |
+
* or a build-time ``VITE_CALL_FULL_DUPLEX_ENABLED``.
|
| 28 |
+
* With the flag off every consumer keeps
|
| 29 |
+
* its pre-flag behaviour β lets us flip
|
| 30 |
+
* the full-duplex fix on/off during
|
| 31 |
+
* dogfood without editing code.
|
| 32 |
+
*
|
| 33 |
+
* No runtime DOM setup; safe to import from SSR contexts β the
|
| 34 |
+
* speakOwned path guards on ``typeof window``.
|
| 35 |
+
*/
|
| 36 |
+
|
| 37 |
+
export type CallLogEvent =
|
| 38 |
+
| { e: 'lifecycle'; phase: 'open' | 'ready' | 'greet' | 'hangup' | 'close'; sid?: string }
|
| 39 |
+
| { e: 'tts'; action: 'start' | 'end' | 'dedupe'; source: 'overlay' | 'app-level'; textLen: number }
|
| 40 |
+
| { e: 'vad'; action: 'speech_start' | 'speech_end' | 'suppressed'; level?: number; state: string }
|
| 41 |
+
| { e: 'turn'; action: 'user_out' | 'assistant_in' | 'barge_in'; route: 'ws' | 'chat_rest'; latencyMs?: number }
|
| 42 |
+
| { e: 'state'; from: string; to: string; trigger: string }
|
| 43 |
+
| { e: 'error'; where: string; message: string }
|
| 44 |
+
|
| 45 |
+
declare global {
|
| 46 |
+
interface Window {
|
| 47 |
+
__hp_tts_owner__?: 'overlay' | 'app-level' | null
|
| 48 |
+
}
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
export function clog(evt: CallLogEvent): void {
|
| 52 |
+
if (typeof performance === 'undefined') return
|
| 53 |
+
const ts = Math.floor(performance.now())
|
| 54 |
+
// console.info so DevTools 'Info' filter can hide it in prod if
|
| 55 |
+
// needed; keeps the log present for debug without noise.
|
| 56 |
+
// eslint-disable-next-line no-console
|
| 57 |
+
console.info(`[Call+${ts}]`, evt)
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
/** Resolve the feature-flag state. Priority:
|
| 61 |
+
* 1. ``localStorage.homepilot_call_full_duplex`` override
|
| 62 |
+
* 2. build-time ``VITE_CALL_FULL_DUPLEX_ENABLED`` (default 'true')
|
| 63 |
+
* Falls through to ``true`` on any read error. */
|
| 64 |
+
export function isCallFullDuplexEnabled(): boolean {
|
| 65 |
+
if (typeof window === 'undefined') return true
|
| 66 |
+
try {
|
| 67 |
+
const override = window.localStorage.getItem('homepilot_call_full_duplex')
|
| 68 |
+
if (override === 'true') return true
|
| 69 |
+
if (override === 'false') return false
|
| 70 |
+
} catch {
|
| 71 |
+
/* ignore storage errors */
|
| 72 |
+
}
|
| 73 |
+
const envVal = (import.meta as unknown as {
|
| 74 |
+
env?: Record<string, string | undefined>
|
| 75 |
+
}).env?.VITE_CALL_FULL_DUPLEX_ENABLED
|
| 76 |
+
return String(envVal ?? 'true') !== 'false'
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
export interface SpeakOwnedHooks {
|
| 80 |
+
onStart?: () => void
|
| 81 |
+
onEnd?: () => void
|
| 82 |
+
onError?: (message: string) => void
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
/**
|
| 86 |
+
* Speak ``text`` via ``window.SpeechService`` with a shared owner
|
| 87 |
+
* lock. Returns ``true`` if this call owns the utterance, ``false``
|
| 88 |
+
* if another owner was already speaking (dedupe). Hooks fire on
|
| 89 |
+
* the owning path only.
|
| 90 |
+
*
|
| 91 |
+
* Fallback when the shim isn't available: emits the ``tts.start``
|
| 92 |
+
* log + an estimated ``tts.end`` timer anyway, so callers still
|
| 93 |
+
* get their ``onEnd`` hook (used by the mic-gate release in
|
| 94 |
+
* ``CallOverlay``) even with no TTS engine present.
|
| 95 |
+
*/
|
| 96 |
+
export function speakOwned(
|
| 97 |
+
source: 'overlay' | 'app-level',
|
| 98 |
+
text: string,
|
| 99 |
+
hooks?: SpeakOwnedHooks,
|
| 100 |
+
): boolean {
|
| 101 |
+
if (typeof window === 'undefined') return false
|
| 102 |
+
const t = (text ?? '').trim()
|
| 103 |
+
if (!t) return false
|
| 104 |
+
const owner = window.__hp_tts_owner__
|
| 105 |
+
if (owner && owner !== source) {
|
| 106 |
+
clog({ e: 'tts', action: 'dedupe', source, textLen: t.length })
|
| 107 |
+
return false
|
| 108 |
+
}
|
| 109 |
+
window.__hp_tts_owner__ = source
|
| 110 |
+
try {
|
| 111 |
+
hooks?.onStart?.()
|
| 112 |
+
clog({ e: 'tts', action: 'start', source, textLen: t.length })
|
| 113 |
+
const svc = (window as unknown as {
|
| 114 |
+
SpeechService?: { speak?: (text: string) => void }
|
| 115 |
+
}).SpeechService
|
| 116 |
+
svc?.speak?.(t)
|
| 117 |
+
// Length-estimate for the onEnd release β the SpeechService shim
|
| 118 |
+
// in this repo doesn't expose an explicit onend hook, so we
|
| 119 |
+
// guesstimate from text length + cap it so a runaway payload
|
| 120 |
+
// can't permanently hold the owner lock. ~12 chars/s typical;
|
| 121 |
+
// [800 ms, 10 s] clamp.
|
| 122 |
+
const estimateMs = Math.max(800, Math.min(10_000, t.length * 80))
|
| 123 |
+
window.setTimeout(() => {
|
| 124 |
+
if (window.__hp_tts_owner__ === source) {
|
| 125 |
+
window.__hp_tts_owner__ = null
|
| 126 |
+
}
|
| 127 |
+
hooks?.onEnd?.()
|
| 128 |
+
clog({ e: 'tts', action: 'end', source, textLen: t.length })
|
| 129 |
+
}, estimateMs)
|
| 130 |
+
return true
|
| 131 |
+
} catch (err) {
|
| 132 |
+
if (window.__hp_tts_owner__ === source) {
|
| 133 |
+
window.__hp_tts_owner__ = null
|
| 134 |
+
}
|
| 135 |
+
const message = err instanceof Error ? err.message : String(err)
|
| 136 |
+
hooks?.onError?.(message)
|
| 137 |
+
clog({ e: 'error', where: 'speakOwned', message })
|
| 138 |
+
return false
|
| 139 |
+
}
|
| 140 |
+
}
|
frontend/src/ui/call/streamTts.test.ts
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Unit tests for streamTts β the pluggable streaming TTS surface.
|
| 3 |
+
* Anchored to Β§ 5.3 of docs/analysis/voice-call-streaming-design.md.
|
| 4 |
+
*/
|
| 5 |
+
import { describe, it, expect } from 'vitest'
|
| 6 |
+
import { createStreamingTts, streamTtsInternals } from './streamTts'
|
| 7 |
+
|
| 8 |
+
const { splitIntoSentences, NullStreamTts } = streamTtsInternals
|
| 9 |
+
|
| 10 |
+
describe('splitIntoSentences', () => {
|
| 11 |
+
it('splits on sentence enders followed by whitespace', () => {
|
| 12 |
+
const { sentences, remainder } = splitIntoSentences(
|
| 13 |
+
'hello there. how are you? fine! and you',
|
| 14 |
+
)
|
| 15 |
+
expect(sentences).toEqual([
|
| 16 |
+
'hello there.',
|
| 17 |
+
'how are you?',
|
| 18 |
+
'fine!',
|
| 19 |
+
])
|
| 20 |
+
expect(remainder).toBe('and you')
|
| 21 |
+
})
|
| 22 |
+
|
| 23 |
+
it('keeps partial trailing text as remainder', () => {
|
| 24 |
+
const { sentences, remainder } = splitIntoSentences('partial sentence')
|
| 25 |
+
expect(sentences).toEqual([])
|
| 26 |
+
expect(remainder).toBe('partial sentence')
|
| 27 |
+
})
|
| 28 |
+
|
| 29 |
+
it('is safe on empty input', () => {
|
| 30 |
+
const { sentences, remainder } = splitIntoSentences('')
|
| 31 |
+
expect(sentences).toEqual([])
|
| 32 |
+
expect(remainder).toBe('')
|
| 33 |
+
})
|
| 34 |
+
|
| 35 |
+
it('splits on newlines + semicolons', () => {
|
| 36 |
+
const { sentences, remainder } = splitIntoSentences(
|
| 37 |
+
'one thing;\n another thing.\n tail',
|
| 38 |
+
)
|
| 39 |
+
// Exact count + order matters so regressions are obvious.
|
| 40 |
+
expect(sentences.length).toBeGreaterThanOrEqual(2)
|
| 41 |
+
expect(sentences[0]).toContain('one thing')
|
| 42 |
+
expect(remainder.trim()).toBe('tail')
|
| 43 |
+
})
|
| 44 |
+
})
|
| 45 |
+
|
| 46 |
+
describe('NullStreamTts fallback', () => {
|
| 47 |
+
it('is safe to call every method; never claims to be speaking', () => {
|
| 48 |
+
const tts = new NullStreamTts()
|
| 49 |
+
expect(tts.isSpeaking).toBe(false)
|
| 50 |
+
tts.appendDelta('hello')
|
| 51 |
+
tts.flush()
|
| 52 |
+
tts.stop()
|
| 53 |
+
expect(tts.isSpeaking).toBe(false)
|
| 54 |
+
})
|
| 55 |
+
})
|
| 56 |
+
|
| 57 |
+
describe('createStreamingTts factory', () => {
|
| 58 |
+
it('returns an object implementing the StreamingTts contract', () => {
|
| 59 |
+
const tts = createStreamingTts()
|
| 60 |
+
expect(typeof tts.appendDelta).toBe('function')
|
| 61 |
+
expect(typeof tts.flush).toBe('function')
|
| 62 |
+
expect(typeof tts.stop).toBe('function')
|
| 63 |
+
expect(typeof tts.isSpeaking).toBe('boolean')
|
| 64 |
+
})
|
| 65 |
+
})
|
frontend/src/ui/call/streamTts.ts
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* streamTts.ts β pluggable streaming text-to-speech surface.
|
| 3 |
+
*
|
| 4 |
+
* One interface, three planned implementations. This file ships the
|
| 5 |
+
* first (browser Web Speech); Piper-WASM and remote vendor streamers
|
| 6 |
+
* are follow-ups that just swap the concrete constructor in
|
| 7 |
+
* :func:`createStreamingTts`.
|
| 8 |
+
*
|
| 9 |
+
* Contract (Β§ 5.3 of docs/analysis/voice-call-streaming-design.md):
|
| 10 |
+
*
|
| 11 |
+
* appendDelta(delta) push a new text chunk. Implementations
|
| 12 |
+
* flush to the audio engine at sentence
|
| 13 |
+
* boundaries; callers hand over raw deltas
|
| 14 |
+
* without worrying about clause shaping.
|
| 15 |
+
* flush() signal end-of-turn. Any residual buffer
|
| 16 |
+
* is spoken; the engine transitions back
|
| 17 |
+
* to idle once the audio drains.
|
| 18 |
+
* stop() barge-in. MUST silence any audio playing
|
| 19 |
+
* or scheduled to play within β€ 50 ms on
|
| 20 |
+
* Chromium. Cheap to call redundantly.
|
| 21 |
+
* isSpeaking read-only; true while audio is playing or
|
| 22 |
+
* scheduled. Drives the VAD-tap enable gate.
|
| 23 |
+
*/
|
| 24 |
+
|
| 25 |
+
export interface StreamingTts {
|
| 26 |
+
appendDelta(delta: string): void
|
| 27 |
+
flush(): void
|
| 28 |
+
stop(): void
|
| 29 |
+
readonly isSpeaking: boolean
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
// ββ Sentence-boundary helper ββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
//
|
| 34 |
+
// Flush on: . ! ? ; or a newline β but only when they're followed
|
| 35 |
+
// by whitespace or end-of-buffer (so "e.g." / "Dr." don't mis-split).
|
| 36 |
+
// A tiny regex, kept readable rather than clever; the failure mode on
|
| 37 |
+
// a miss is "one extra utterance per turn," not a correctness bug.
|
| 38 |
+
|
| 39 |
+
const SENTENCE_END = /([.!?;\n])\s+/g
|
| 40 |
+
|
| 41 |
+
function splitIntoSentences(buffer: string): {
|
| 42 |
+
sentences: string[]
|
| 43 |
+
remainder: string
|
| 44 |
+
} {
|
| 45 |
+
const sentences: string[] = []
|
| 46 |
+
let lastIndex = 0
|
| 47 |
+
let m: RegExpExecArray | null
|
| 48 |
+
SENTENCE_END.lastIndex = 0
|
| 49 |
+
while ((m = SENTENCE_END.exec(buffer)) !== null) {
|
| 50 |
+
const end = m.index + m[0].length
|
| 51 |
+
const piece = buffer.slice(lastIndex, end).trim()
|
| 52 |
+
if (piece) sentences.push(piece)
|
| 53 |
+
lastIndex = end
|
| 54 |
+
}
|
| 55 |
+
const remainder = buffer.slice(lastIndex)
|
| 56 |
+
return { sentences, remainder }
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
// ββ Web Speech implementation (ships first) ββββββββββββββββββββββββββ
|
| 60 |
+
//
|
| 61 |
+
// Uses ``window.speechSynthesis`` directly. Each sentence becomes
|
| 62 |
+
// exactly one SpeechSynthesisUtterance so barge-in (via ``cancel()``)
|
| 63 |
+
// drops the current sentence + the queue atomically.
|
| 64 |
+
//
|
| 65 |
+
// Why one utterance per sentence instead of one per delta:
|
| 66 |
+
//
|
| 67 |
+
// 1. SpeechSynthesis queues utterances, not characters. Passing every
|
| 68 |
+
// delta through would create 40+ queued utterances per turn on
|
| 69 |
+
// Chromium, each with its own intonation reset β sounds robotic.
|
| 70 |
+
// 2. Cancel semantics are per-utterance, so sentence-granularity
|
| 71 |
+
// means barge-in can interrupt mid-sentence without stranding
|
| 72 |
+
// half-spoken buffers.
|
| 73 |
+
|
| 74 |
+
class WebSpeechStreamTts implements StreamingTts {
|
| 75 |
+
private buffer = ''
|
| 76 |
+
private speakingCount = 0
|
| 77 |
+
private stopped = false
|
| 78 |
+
private readonly syn: SpeechSynthesis | null
|
| 79 |
+
|
| 80 |
+
constructor() {
|
| 81 |
+
this.syn =
|
| 82 |
+
typeof window !== 'undefined' && 'speechSynthesis' in window
|
| 83 |
+
? window.speechSynthesis
|
| 84 |
+
: null
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
get isSpeaking(): boolean {
|
| 88 |
+
return this.speakingCount > 0
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
appendDelta(delta: string): void {
|
| 92 |
+
if (this.stopped || !delta) return
|
| 93 |
+
this.buffer += delta
|
| 94 |
+
const { sentences, remainder } = splitIntoSentences(this.buffer)
|
| 95 |
+
this.buffer = remainder
|
| 96 |
+
for (const s of sentences) this._speak(s)
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
flush(): void {
|
| 100 |
+
if (this.stopped) return
|
| 101 |
+
if (this.buffer.trim()) {
|
| 102 |
+
this._speak(this.buffer.trim())
|
| 103 |
+
this.buffer = ''
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
stop(): void {
|
| 108 |
+
// Idempotent β safe to call repeatedly from the barge-in path.
|
| 109 |
+
this.stopped = true
|
| 110 |
+
this.buffer = ''
|
| 111 |
+
this.speakingCount = 0
|
| 112 |
+
try {
|
| 113 |
+
this.syn?.cancel()
|
| 114 |
+
} catch {
|
| 115 |
+
/* cancel is best-effort */
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
private _speak(text: string): void {
|
| 120 |
+
if (!this.syn) return
|
| 121 |
+
let utter: SpeechSynthesisUtterance
|
| 122 |
+
try {
|
| 123 |
+
utter = new SpeechSynthesisUtterance(text)
|
| 124 |
+
} catch {
|
| 125 |
+
return
|
| 126 |
+
}
|
| 127 |
+
// Track speaking state so the VAD-tap enable gate stays tight
|
| 128 |
+
// even when the queue reorders utterances (Safari / old Chromium).
|
| 129 |
+
this.speakingCount += 1
|
| 130 |
+
const done = () => {
|
| 131 |
+
this.speakingCount = Math.max(0, this.speakingCount - 1)
|
| 132 |
+
}
|
| 133 |
+
utter.onend = done
|
| 134 |
+
utter.onerror = done
|
| 135 |
+
try {
|
| 136 |
+
this.syn.speak(utter)
|
| 137 |
+
} catch {
|
| 138 |
+
done()
|
| 139 |
+
}
|
| 140 |
+
}
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
// ββ Null implementation β used in tests + as a safety fallback βββββββ
|
| 144 |
+
|
| 145 |
+
class NullStreamTts implements StreamingTts {
|
| 146 |
+
isSpeaking = false
|
| 147 |
+
appendDelta(_delta: string): void { /* no-op */ }
|
| 148 |
+
flush(): void { /* no-op */ }
|
| 149 |
+
stop(): void { /* no-op */ }
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
// ββ Factory βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 153 |
+
//
|
| 154 |
+
// Returns the best implementation available at runtime. Swapping
|
| 155 |
+
// in Piper-WASM or a remote vendor streamer is a one-line change
|
| 156 |
+
// in this function β call sites don't need to know.
|
| 157 |
+
|
| 158 |
+
export function createStreamingTts(): StreamingTts {
|
| 159 |
+
if (typeof window === 'undefined') return new NullStreamTts()
|
| 160 |
+
if (!('speechSynthesis' in window)) return new NullStreamTts()
|
| 161 |
+
// Probe for the existence of an app-level SpeechService shim; if
|
| 162 |
+
// present we could route through it, but for now the direct Web
|
| 163 |
+
// Speech path is both simpler and lower-latency.
|
| 164 |
+
return new WebSpeechStreamTts()
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
// Exported for tests + rare callers that want an explicit unplugged
|
| 168 |
+
// implementation (e.g. when a user opts out of TTS entirely).
|
| 169 |
+
export const streamTtsInternals = {
|
| 170 |
+
splitIntoSentences,
|
| 171 |
+
WebSpeechStreamTts,
|
| 172 |
+
NullStreamTts,
|
| 173 |
+
}
|
frontend/src/ui/call/useCallSession.ts
ADDED
|
@@ -0,0 +1,381 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* useCallSession β React hook that owns the voice_call lifecycle.
|
| 3 |
+
*
|
| 4 |
+
* One instance per mounted `CallOverlayInner`. Composes callApi +
|
| 5 |
+
* callSocket into a single, cleanup-safe React surface:
|
| 6 |
+
*
|
| 7 |
+
* const session = useCallSession({ enabled, backendUrl, authToken,
|
| 8 |
+
* conversationId, personaId,
|
| 9 |
+
* deviceInfo })
|
| 10 |
+
*
|
| 11 |
+
* session.status β 'idle' | 'creating' | 'connecting'
|
| 12 |
+
* | 'live' | 'reconnecting'
|
| 13 |
+
* | 'unavailable' | 'error' | 'closed'
|
| 14 |
+
* session.sendTranscript(text) β route STT capture to the backend
|
| 15 |
+
* session.end() β graceful user-initiated close
|
| 16 |
+
* session.onAssistantTranscript β subscribe to assistant replies
|
| 17 |
+
* session.onAssistantFiller β subscribe to "hmmβ¦" events
|
| 18 |
+
* session.onAssistantBackchannel β subscribe to "mm-hm" events
|
| 19 |
+
* session.lastError β surface for the UI
|
| 20 |
+
*
|
| 21 |
+
* When `enabled` flips to true we:
|
| 22 |
+
* 1. POST /v1/voice-call/sessions (via callApi)
|
| 23 |
+
* 2. Open the WebSocket (via callSocket)
|
| 24 |
+
* 3. Wire status + event forwarding into React state
|
| 25 |
+
*
|
| 26 |
+
* Critically, if the POST returns 404/501 (backend has
|
| 27 |
+
* VOICE_CALL_ENABLED=false) we surface `status='unavailable'` and
|
| 28 |
+
* stop β the caller then falls back to the chat-REST path. That
|
| 29 |
+
* graceful-degradation path is what lets us ship this module
|
| 30 |
+
* independent of the backend flag rollout.
|
| 31 |
+
*/
|
| 32 |
+
|
| 33 |
+
import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
| 34 |
+
import {
|
| 35 |
+
createCallSession,
|
| 36 |
+
resolveWsUrl,
|
| 37 |
+
CallApiError,
|
| 38 |
+
type CreateCallSessionRequest,
|
| 39 |
+
type CreateCallSessionResponse,
|
| 40 |
+
} from './callApi'
|
| 41 |
+
import {
|
| 42 |
+
CallSocket,
|
| 43 |
+
type AssistantBackchannelPayload,
|
| 44 |
+
type AssistantCancelPayload,
|
| 45 |
+
type AssistantFillerPayload,
|
| 46 |
+
type AssistantPartialPayload,
|
| 47 |
+
type AssistantTranscriptPayload,
|
| 48 |
+
type AssistantTurnEndPayload,
|
| 49 |
+
type CallCloseReason,
|
| 50 |
+
type CallLifecycleStatus,
|
| 51 |
+
} from './callSocket'
|
| 52 |
+
|
| 53 |
+
export type CallSessionStatus =
|
| 54 |
+
| 'idle'
|
| 55 |
+
| 'creating'
|
| 56 |
+
| 'connecting'
|
| 57 |
+
| 'live'
|
| 58 |
+
| 'reconnecting'
|
| 59 |
+
| 'draining'
|
| 60 |
+
| 'closed'
|
| 61 |
+
| 'unavailable'
|
| 62 |
+
| 'error'
|
| 63 |
+
|
| 64 |
+
export interface UseCallSessionArgs {
|
| 65 |
+
/** The hook is inert until this flips to true. Flipping back to
|
| 66 |
+
* false triggers a graceful dispose. */
|
| 67 |
+
enabled: boolean
|
| 68 |
+
/** Resolved HomePilot backend URL (scheme + host). */
|
| 69 |
+
backendUrl: string
|
| 70 |
+
/** Bearer JWT for the REST handshake + WS query param. */
|
| 71 |
+
authToken: string | null
|
| 72 |
+
/** Session-creation request fields. */
|
| 73 |
+
request: CreateCallSessionRequest
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
export interface CallSessionHandle {
|
| 77 |
+
status: CallSessionStatus
|
| 78 |
+
/** Server-side canonical status last seen (live / ending / ended). */
|
| 79 |
+
callState: 'live' | 'ending' | 'ended' | null
|
| 80 |
+
/** Raw close reason once the session terminates. Useful for
|
| 81 |
+
* differentiating user-end vs timeout vs backend-disabled. */
|
| 82 |
+
closeReason: CallCloseReason | null
|
| 83 |
+
/** Any terminal error. 'unavailable' is NOT an error β it's a
|
| 84 |
+
* legitimate graceful-degradation signal. */
|
| 85 |
+
lastError: string | null
|
| 86 |
+
/** Route a final STT transcript into the call turn. No-op outside
|
| 87 |
+
* the 'live' | 'reconnecting' states. */
|
| 88 |
+
sendTranscript: (text: string) => void
|
| 89 |
+
/** Graceful end β sends call.control end, closes the socket. */
|
| 90 |
+
end: () => void
|
| 91 |
+
/** Bookkeeping. Server persists the *type* of the event only. */
|
| 92 |
+
sendUiState: (p: { muted?: boolean; speaker_on?: boolean; backgrounded?: boolean }) => void
|
| 93 |
+
/** Phase 2 β interim user STT (secondary barge-in signal). */
|
| 94 |
+
sendTranscriptPartial: (p: { text: string; stable_prefix_len?: number }) => void
|
| 95 |
+
/** Phase 3 β explicit user interrupt carrying the active turn_id. */
|
| 96 |
+
sendBargeIn: (turn_id: string) => void
|
| 97 |
+
/** Whether this session negotiated streaming. Pulled from the
|
| 98 |
+
* session-create capabilities response. False = unary mode. */
|
| 99 |
+
streamingNegotiated: boolean
|
| 100 |
+
/** Whether this session negotiated barge-in (implies streaming). */
|
| 101 |
+
bargeInNegotiated: boolean
|
| 102 |
+
/** Subscribe helpers β each returns an unsubscribe fn. Stable
|
| 103 |
+
* identities via useRef-backed implementation so the caller can
|
| 104 |
+
* pass them to useEffect deps without re-subscription storms. */
|
| 105 |
+
onAssistantTranscript: (fn: (p: AssistantTranscriptPayload) => void) => () => void
|
| 106 |
+
onAssistantFiller: (fn: (p: AssistantFillerPayload) => void) => () => void
|
| 107 |
+
onAssistantBackchannel: (fn: (p: AssistantBackchannelPayload) => void) => () => void
|
| 108 |
+
/** Phase 2 subscribes. */
|
| 109 |
+
onAssistantPartial: (fn: (p: AssistantPartialPayload) => void) => () => void
|
| 110 |
+
onAssistantTurnEnd: (fn: (p: AssistantTurnEndPayload) => void) => () => void
|
| 111 |
+
onAssistantCancel: (fn: (p: AssistantCancelPayload) => void) => () => void
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
// When the backend returns 404/501 for voice_call (VOICE_CALL_ENABLED=false)
|
| 115 |
+
// we remember the miss per-backend and short-circuit future session creates
|
| 116 |
+
// for UNAVAILABLE_BACKOFF_MS. Without this, every CallOverlay mount posts a
|
| 117 |
+
// fresh 404 against the same flag, spamming the console and delaying the
|
| 118 |
+
// fallback path by a full network round-trip.
|
| 119 |
+
const UNAVAILABLE_BACKOFF_MS = 30_000
|
| 120 |
+
const unavailableUntilByBackend = new Map<string, number>()
|
| 121 |
+
|
| 122 |
+
export function useCallSession(args: UseCallSessionArgs): CallSessionHandle {
|
| 123 |
+
const { enabled, backendUrl, authToken, request } = args
|
| 124 |
+
|
| 125 |
+
const [status, setStatus] = useState<CallSessionStatus>('idle')
|
| 126 |
+
const [callState, setCallState] = useState<'live' | 'ending' | 'ended' | null>(null)
|
| 127 |
+
const [closeReason, setCloseReason] = useState<CallCloseReason | null>(null)
|
| 128 |
+
const [lastError, setLastError] = useState<string | null>(null)
|
| 129 |
+
|
| 130 |
+
const socketRef = useRef<CallSocket | null>(null)
|
| 131 |
+
const sessionRef = useRef<CreateCallSessionResponse | null>(null)
|
| 132 |
+
|
| 133 |
+
// Stable subscribe surface. Keep listener sets here so they survive
|
| 134 |
+
// socket reconnects β the old socket's listeners are torn down with
|
| 135 |
+
// it, but user subscriptions persist across reconnects and are
|
| 136 |
+
// re-attached when a new socket comes up.
|
| 137 |
+
const txListeners = useRef(new Set<(p: AssistantTranscriptPayload) => void>())
|
| 138 |
+
const fillerListeners = useRef(new Set<(p: AssistantFillerPayload) => void>())
|
| 139 |
+
const bcListeners = useRef(new Set<(p: AssistantBackchannelPayload) => void>())
|
| 140 |
+
// Phase 2/3 listener sets. Kept separate so an old subscribe set
|
| 141 |
+
// doesn't churn when a new one mounts (stream wire-up rebinds often
|
| 142 |
+
// during CallOverlay renders).
|
| 143 |
+
const partialListeners = useRef(new Set<(p: AssistantPartialPayload) => void>())
|
| 144 |
+
const turnEndListeners = useRef(new Set<(p: AssistantTurnEndPayload) => void>())
|
| 145 |
+
const cancelListeners = useRef(new Set<(p: AssistantCancelPayload) => void>())
|
| 146 |
+
// Negotiated modes β re-derived on session create, stashed here so
|
| 147 |
+
// every render reads the same value without re-running the effect.
|
| 148 |
+
const [streamingNegotiated, setStreamingNegotiated] = useState(false)
|
| 149 |
+
const [bargeInNegotiated, setBargeInNegotiated] = useState(false)
|
| 150 |
+
|
| 151 |
+
// Ref'd primitive so the session-create effect doesn't re-trigger
|
| 152 |
+
// when the request object identity changes every render.
|
| 153 |
+
const requestRef = useRef(request)
|
| 154 |
+
useEffect(() => { requestRef.current = request }, [request])
|
| 155 |
+
|
| 156 |
+
// Main lifecycle. Runs once per (enabled, backendUrl, authToken)
|
| 157 |
+
// triple β which is what we want: flipping enabled true/false
|
| 158 |
+
// drives open/close; swapping backendUrl or re-auth reconnects.
|
| 159 |
+
useEffect(() => {
|
| 160 |
+
if (!enabled) return
|
| 161 |
+
|
| 162 |
+
let disposed = false
|
| 163 |
+
const teardown = () => {
|
| 164 |
+
disposed = true
|
| 165 |
+
socketRef.current?.dispose('unmounted')
|
| 166 |
+
socketRef.current = null
|
| 167 |
+
sessionRef.current = null
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
const run = async () => {
|
| 171 |
+
setStatus('creating')
|
| 172 |
+
setLastError(null)
|
| 173 |
+
setCloseReason(null)
|
| 174 |
+
setCallState(null)
|
| 175 |
+
|
| 176 |
+
// Skip the POST entirely if this backend has been 404/501-ing
|
| 177 |
+
// recently. Saves a round-trip per call re-entry while the flag
|
| 178 |
+
// is off server-side; the backoff window expires automatically
|
| 179 |
+
// when we clear the map entry below on a success.
|
| 180 |
+
const skipUntil = unavailableUntilByBackend.get(backendUrl)
|
| 181 |
+
if (skipUntil && Date.now() < skipUntil) {
|
| 182 |
+
// eslint-disable-next-line no-console
|
| 183 |
+
console.info(
|
| 184 |
+
'[useCallSession] skipping createCallSession β backend flagged unavailable',
|
| 185 |
+
{ backendUrl, resumesAt: new Date(skipUntil).toISOString() },
|
| 186 |
+
)
|
| 187 |
+
setStatus('unavailable')
|
| 188 |
+
return
|
| 189 |
+
}
|
| 190 |
+
|
| 191 |
+
let handshake: CreateCallSessionResponse
|
| 192 |
+
try {
|
| 193 |
+
handshake = await createCallSession(backendUrl, requestRef.current, authToken)
|
| 194 |
+
} catch (err) {
|
| 195 |
+
if (disposed) return
|
| 196 |
+
if (err instanceof CallApiError && err.isUnavailable) {
|
| 197 |
+
unavailableUntilByBackend.set(
|
| 198 |
+
backendUrl,
|
| 199 |
+
Date.now() + UNAVAILABLE_BACKOFF_MS,
|
| 200 |
+
)
|
| 201 |
+
// eslint-disable-next-line no-console
|
| 202 |
+
console.warn(
|
| 203 |
+
'[useCallSession] voice_call unavailable β falling back to chat REST for',
|
| 204 |
+
`${UNAVAILABLE_BACKOFF_MS}ms`,
|
| 205 |
+
)
|
| 206 |
+
setStatus('unavailable')
|
| 207 |
+
return
|
| 208 |
+
}
|
| 209 |
+
// eslint-disable-next-line no-console
|
| 210 |
+
console.error('[useCallSession] createCallSession failed', err)
|
| 211 |
+
setStatus('error')
|
| 212 |
+
setLastError(err instanceof Error ? err.message : String(err))
|
| 213 |
+
return
|
| 214 |
+
}
|
| 215 |
+
if (disposed) return
|
| 216 |
+
|
| 217 |
+
// Handshake worked β clear any prior backoff so new flag flips
|
| 218 |
+
// take effect immediately instead of waiting out the window.
|
| 219 |
+
unavailableUntilByBackend.delete(backendUrl)
|
| 220 |
+
|
| 221 |
+
sessionRef.current = handshake
|
| 222 |
+
const url = resolveWsUrl(
|
| 223 |
+
handshake.ws_url,
|
| 224 |
+
handshake.session_id,
|
| 225 |
+
handshake.resume_token,
|
| 226 |
+
authToken,
|
| 227 |
+
backendUrl,
|
| 228 |
+
)
|
| 229 |
+
|
| 230 |
+
const sock = new CallSocket({ url })
|
| 231 |
+
socketRef.current = sock
|
| 232 |
+
|
| 233 |
+
// Translate socket lifecycle β hook status.
|
| 234 |
+
sock.on('statusChange', (s: CallLifecycleStatus) => {
|
| 235 |
+
if (disposed) return
|
| 236 |
+
setStatus(s === 'idle' ? 'idle' : s)
|
| 237 |
+
})
|
| 238 |
+
sock.on('callState', (p) => {
|
| 239 |
+
if (disposed) return
|
| 240 |
+
setCallState(p.status)
|
| 241 |
+
})
|
| 242 |
+
sock.on('closed', ({ reason }) => {
|
| 243 |
+
if (disposed) return
|
| 244 |
+
setStatus('closed')
|
| 245 |
+
setCloseReason(reason)
|
| 246 |
+
})
|
| 247 |
+
sock.on('serverError', (p) => {
|
| 248 |
+
// Non-terminal by contract; surface for the UI but keep the
|
| 249 |
+
// socket open. The server will close explicitly if fatal.
|
| 250 |
+
if (disposed) return
|
| 251 |
+
setLastError(`${p.code}: ${p.message}`)
|
| 252 |
+
})
|
| 253 |
+
sock.on('assistantTranscript', (p) => {
|
| 254 |
+
for (const fn of txListeners.current) fn(p)
|
| 255 |
+
})
|
| 256 |
+
sock.on('assistantFiller', (p) => {
|
| 257 |
+
for (const fn of fillerListeners.current) fn(p)
|
| 258 |
+
})
|
| 259 |
+
sock.on('assistantBackchannel', (p) => {
|
| 260 |
+
for (const fn of bcListeners.current) fn(p)
|
| 261 |
+
})
|
| 262 |
+
// Phase 2/3 event fan-outs.
|
| 263 |
+
sock.on('assistantPartial', (p) => {
|
| 264 |
+
for (const fn of partialListeners.current) fn(p)
|
| 265 |
+
})
|
| 266 |
+
sock.on('assistantTurnEnd', (p) => {
|
| 267 |
+
for (const fn of turnEndListeners.current) fn(p)
|
| 268 |
+
})
|
| 269 |
+
sock.on('assistantCancel', (p) => {
|
| 270 |
+
for (const fn of cancelListeners.current) fn(p)
|
| 271 |
+
})
|
| 272 |
+
|
| 273 |
+
// Reflect negotiated modes from the session-create response.
|
| 274 |
+
const caps = handshake.capabilities as
|
| 275 |
+
| { streaming?: boolean; barge_in?: boolean } | undefined
|
| 276 |
+
setStreamingNegotiated(!!caps?.streaming)
|
| 277 |
+
setBargeInNegotiated(!!caps?.streaming && !!caps?.barge_in)
|
| 278 |
+
|
| 279 |
+
sock.connect()
|
| 280 |
+
}
|
| 281 |
+
|
| 282 |
+
void run()
|
| 283 |
+
return teardown
|
| 284 |
+
}, [enabled, backendUrl, authToken])
|
| 285 |
+
|
| 286 |
+
// Public actions. Guarded against stale-socket calls. --------------
|
| 287 |
+
|
| 288 |
+
const sendTranscript = useCallback((text: string) => {
|
| 289 |
+
const sock = socketRef.current
|
| 290 |
+
if (!sock) return
|
| 291 |
+
const trimmed = text.trim()
|
| 292 |
+
if (!trimmed) return
|
| 293 |
+
sock.sendTranscript({ text: trimmed })
|
| 294 |
+
}, [])
|
| 295 |
+
|
| 296 |
+
const end = useCallback(() => {
|
| 297 |
+
socketRef.current?.end()
|
| 298 |
+
}, [])
|
| 299 |
+
|
| 300 |
+
const sendUiState = useCallback(
|
| 301 |
+
(p: { muted?: boolean; speaker_on?: boolean; backgrounded?: boolean }) => {
|
| 302 |
+
socketRef.current?.sendUiState(p)
|
| 303 |
+
},
|
| 304 |
+
[],
|
| 305 |
+
)
|
| 306 |
+
|
| 307 |
+
const sendTranscriptPartial = useCallback(
|
| 308 |
+
(p: { text: string; stable_prefix_len?: number }) => {
|
| 309 |
+
socketRef.current?.sendTranscriptPartial(p)
|
| 310 |
+
},
|
| 311 |
+
[],
|
| 312 |
+
)
|
| 313 |
+
|
| 314 |
+
const sendBargeIn = useCallback((turn_id: string) => {
|
| 315 |
+
socketRef.current?.sendBargeIn(turn_id)
|
| 316 |
+
}, [])
|
| 317 |
+
|
| 318 |
+
const onAssistantTranscript = useCallback(
|
| 319 |
+
(fn: (p: AssistantTranscriptPayload) => void) => {
|
| 320 |
+
txListeners.current.add(fn)
|
| 321 |
+
return () => { txListeners.current.delete(fn) }
|
| 322 |
+
}, [],
|
| 323 |
+
)
|
| 324 |
+
const onAssistantFiller = useCallback(
|
| 325 |
+
(fn: (p: AssistantFillerPayload) => void) => {
|
| 326 |
+
fillerListeners.current.add(fn)
|
| 327 |
+
return () => { fillerListeners.current.delete(fn) }
|
| 328 |
+
}, [],
|
| 329 |
+
)
|
| 330 |
+
const onAssistantBackchannel = useCallback(
|
| 331 |
+
(fn: (p: AssistantBackchannelPayload) => void) => {
|
| 332 |
+
bcListeners.current.add(fn)
|
| 333 |
+
return () => { bcListeners.current.delete(fn) }
|
| 334 |
+
}, [],
|
| 335 |
+
)
|
| 336 |
+
const onAssistantPartial = useCallback(
|
| 337 |
+
(fn: (p: AssistantPartialPayload) => void) => {
|
| 338 |
+
partialListeners.current.add(fn)
|
| 339 |
+
return () => { partialListeners.current.delete(fn) }
|
| 340 |
+
}, [],
|
| 341 |
+
)
|
| 342 |
+
const onAssistantTurnEnd = useCallback(
|
| 343 |
+
(fn: (p: AssistantTurnEndPayload) => void) => {
|
| 344 |
+
turnEndListeners.current.add(fn)
|
| 345 |
+
return () => { turnEndListeners.current.delete(fn) }
|
| 346 |
+
}, [],
|
| 347 |
+
)
|
| 348 |
+
const onAssistantCancel = useCallback(
|
| 349 |
+
(fn: (p: AssistantCancelPayload) => void) => {
|
| 350 |
+
cancelListeners.current.add(fn)
|
| 351 |
+
return () => { cancelListeners.current.delete(fn) }
|
| 352 |
+
}, [],
|
| 353 |
+
)
|
| 354 |
+
|
| 355 |
+
return useMemo<CallSessionHandle>(() => ({
|
| 356 |
+
status,
|
| 357 |
+
callState,
|
| 358 |
+
closeReason,
|
| 359 |
+
lastError,
|
| 360 |
+
sendTranscript,
|
| 361 |
+
end,
|
| 362 |
+
sendUiState,
|
| 363 |
+
sendTranscriptPartial,
|
| 364 |
+
sendBargeIn,
|
| 365 |
+
streamingNegotiated,
|
| 366 |
+
bargeInNegotiated,
|
| 367 |
+
onAssistantTranscript,
|
| 368 |
+
onAssistantFiller,
|
| 369 |
+
onAssistantBackchannel,
|
| 370 |
+
onAssistantPartial,
|
| 371 |
+
onAssistantTurnEnd,
|
| 372 |
+
onAssistantCancel,
|
| 373 |
+
}), [
|
| 374 |
+
status, callState, closeReason, lastError,
|
| 375 |
+
sendTranscript, end, sendUiState,
|
| 376 |
+
sendTranscriptPartial, sendBargeIn,
|
| 377 |
+
streamingNegotiated, bargeInNegotiated,
|
| 378 |
+
onAssistantTranscript, onAssistantFiller, onAssistantBackchannel,
|
| 379 |
+
onAssistantPartial, onAssistantTurnEnd, onAssistantCancel,
|
| 380 |
+
])
|
| 381 |
+
}
|
frontend/src/ui/components/CreatorExportWizard.tsx
CHANGED
|
@@ -1,28 +1,65 @@
|
|
| 1 |
/**
|
| 2 |
-
* CreatorExportWizard β
|
|
|
|
| 3 |
*
|
| 4 |
* Flow:
|
| 5 |
-
* 1.
|
| 6 |
-
* 2.
|
| 7 |
-
*
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
*
|
| 9 |
* Race protection:
|
| 10 |
-
*
|
| 11 |
-
*
|
| 12 |
-
*
|
| 13 |
-
* session-epoch pattern used in App.tsx for account-switch races.
|
| 14 |
*/
|
| 15 |
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
| 16 |
-
import {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
|
| 18 |
export type ExportKind = 'mp4_plain' | 'mp4_youtube'
|
| 19 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
interface Props {
|
| 21 |
open: boolean
|
| 22 |
onClose: () => void
|
| 23 |
backendUrl: string
|
| 24 |
videoId: string
|
| 25 |
-
/**
|
|
|
|
|
|
|
| 26 |
videoTitle?: string
|
| 27 |
/** Disable submit when there are no scenes or scenes are still generating. */
|
| 28 |
disabledReason?: string | null
|
|
@@ -40,6 +77,7 @@ interface JobView {
|
|
| 40 |
|
| 41 |
const POLL_INTERVAL_MS = 1000
|
| 42 |
const POLL_TIMEOUT_MS = 10 * 60 * 1000 // 10 minutes
|
|
|
|
| 43 |
|
| 44 |
function authHeaders(): Record<string, string> {
|
| 45 |
const token = localStorage.getItem('homepilot_auth_token') || ''
|
|
@@ -50,35 +88,74 @@ function trimBackend(u: string): string {
|
|
| 50 |
return u.replace(/\/+$/, '')
|
| 51 |
}
|
| 52 |
|
|
|
|
|
|
|
| 53 |
export default function CreatorExportWizard({
|
| 54 |
open,
|
| 55 |
onClose,
|
| 56 |
backendUrl,
|
| 57 |
videoId,
|
|
|
|
| 58 |
videoTitle,
|
| 59 |
disabledReason,
|
| 60 |
}: Props) {
|
| 61 |
-
|
|
|
|
|
|
|
|
|
|
| 62 |
const [kind, setKind] = useState<ExportKind>('mp4_youtube')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
const [job, setJob] = useState<JobView | null>(null)
|
| 64 |
const [submitError, setSubmitError] = useState<string | null>(null)
|
| 65 |
|
| 66 |
-
// Race protection
|
| 67 |
-
// the current value, and the poller drops responses whose epoch < current.
|
| 68 |
const epochRef = useRef(0)
|
| 69 |
|
| 70 |
-
// Reset modal
|
| 71 |
useEffect(() => {
|
| 72 |
if (open) {
|
| 73 |
setStep(1)
|
| 74 |
setJob(null)
|
| 75 |
setSubmitError(null)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
} else {
|
| 77 |
-
// Closing the modal invalidates any in-flight poll.
|
| 78 |
epochRef.current += 1
|
|
|
|
| 79 |
}
|
| 80 |
}, [open])
|
| 81 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
const downloadUrl = useMemo(() => {
|
| 83 |
if (!job || job.status !== 'done') return ''
|
| 84 |
const tok = localStorage.getItem('homepilot_auth_token') || ''
|
|
@@ -86,12 +163,115 @@ export default function CreatorExportWizard({
|
|
| 86 |
return `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${job.id}/download${qp}`
|
| 87 |
}, [job, backendUrl, videoId])
|
| 88 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
const pollJob = useCallback(
|
| 90 |
async (jobId: string, myEpoch: number) => {
|
| 91 |
const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${jobId}`
|
| 92 |
const start = Date.now()
|
| 93 |
-
// Polling loop with a hard upper bound.
|
| 94 |
-
// We bail on epoch mismatch so closing the modal cancels the loop.
|
| 95 |
while (Date.now() - start < POLL_TIMEOUT_MS) {
|
| 96 |
if (epochRef.current !== myEpoch) return
|
| 97 |
try {
|
|
@@ -124,8 +304,6 @@ export default function CreatorExportWizard({
|
|
| 124 |
}
|
| 125 |
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
|
| 126 |
}
|
| 127 |
-
// Timed out: surface to the user, leave the job alone so they can
|
| 128 |
-
// re-open the wizard later if it does eventually finish.
|
| 129 |
if (epochRef.current === myEpoch) {
|
| 130 |
setJob((prev) =>
|
| 131 |
prev ? { ...prev, status: 'error', error: 'Render timed out' } : prev,
|
|
@@ -135,50 +313,57 @@ export default function CreatorExportWizard({
|
|
| 135 |
[backendUrl, videoId],
|
| 136 |
)
|
| 137 |
|
| 138 |
-
const
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
}
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
id: j.id,
|
| 165 |
-
status: j.status,
|
| 166 |
-
progress: typeof j.progress === 'number' ? j.progress : 0,
|
| 167 |
-
kind: j.kind,
|
| 168 |
-
})
|
| 169 |
-
void pollJob(j.id, myEpoch)
|
| 170 |
-
} catch (err: any) {
|
| 171 |
-
if (epochRef.current !== myEpoch) return
|
| 172 |
-
const msg = String(err?.message || err)
|
| 173 |
-
setSubmitError(msg)
|
| 174 |
-
setJob({ id: '', status: 'error', progress: 0, error: msg, kind })
|
| 175 |
-
}
|
| 176 |
-
}, [backendUrl, videoId, kind, pollJob])
|
| 177 |
|
| 178 |
if (!open) return null
|
| 179 |
|
| 180 |
const submitDisabled = Boolean(disabledReason)
|
| 181 |
|
|
|
|
|
|
|
| 182 |
return (
|
| 183 |
<div
|
| 184 |
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
|
|
@@ -188,16 +373,26 @@ export default function CreatorExportWizard({
|
|
| 188 |
aria-label="Export video"
|
| 189 |
>
|
| 190 |
<div
|
| 191 |
-
className="w-full max-w-
|
| 192 |
onClick={(e) => e.stopPropagation()}
|
| 193 |
>
|
| 194 |
-
|
|
|
|
| 195 |
<div>
|
| 196 |
<div className="text-sm font-semibold">Export video</div>
|
| 197 |
{videoTitle ? (
|
| 198 |
<div className="text-xs text-white/50 truncate max-w-[28rem]">{videoTitle}</div>
|
| 199 |
) : null}
|
| 200 |
</div>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
<button
|
| 202 |
type="button"
|
| 203 |
onClick={onClose}
|
|
@@ -208,7 +403,8 @@ export default function CreatorExportWizard({
|
|
| 208 |
</button>
|
| 209 |
</div>
|
| 210 |
|
| 211 |
-
|
|
|
|
| 212 |
{step === 1 && (
|
| 213 |
<>
|
| 214 |
<div className="text-xs text-white/60">
|
|
@@ -282,7 +478,7 @@ export default function CreatorExportWizard({
|
|
| 282 |
</button>
|
| 283 |
<button
|
| 284 |
type="button"
|
| 285 |
-
onClick={
|
| 286 |
disabled={submitDisabled}
|
| 287 |
className={[
|
| 288 |
'px-3 py-1.5 rounded-lg text-sm font-medium',
|
|
@@ -291,13 +487,172 @@ export default function CreatorExportWizard({
|
|
| 291 |
: 'bg-white text-black hover:bg-white/90',
|
| 292 |
].join(' ')}
|
| 293 |
>
|
| 294 |
-
|
| 295 |
</button>
|
| 296 |
</div>
|
| 297 |
</>
|
| 298 |
)}
|
| 299 |
|
| 300 |
{step === 2 && (
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 301 |
<>
|
| 302 |
{job?.status === 'done' ? (
|
| 303 |
<div className="space-y-4">
|
|
@@ -315,6 +670,7 @@ export default function CreatorExportWizard({
|
|
| 315 |
</a>
|
| 316 |
<div className="text-[11px] text-white/40 text-center">
|
| 317 |
{job.kind === 'mp4_youtube' ? 'YouTube-optimized' : 'Plain MP4'}
|
|
|
|
| 318 |
</div>
|
| 319 |
</div>
|
| 320 |
) : job?.status === 'error' ? (
|
|
@@ -330,10 +686,10 @@ export default function CreatorExportWizard({
|
|
| 330 |
</div>
|
| 331 |
<button
|
| 332 |
type="button"
|
| 333 |
-
onClick={() => setStep(
|
| 334 |
className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
|
| 335 |
>
|
| 336 |
-
|
| 337 |
</button>
|
| 338 |
</div>
|
| 339 |
) : (
|
|
@@ -347,7 +703,9 @@ export default function CreatorExportWizard({
|
|
| 347 |
<div className="h-2 rounded-full bg-white/10 overflow-hidden">
|
| 348 |
<div
|
| 349 |
className="h-full bg-cyan-400 transition-all"
|
| 350 |
-
style={{
|
|
|
|
|
|
|
| 351 |
/>
|
| 352 |
</div>
|
| 353 |
<div className="text-[11px] text-white/40 text-center">
|
|
|
|
| 1 |
/**
|
| 2 |
+
* CreatorExportWizard β 4-step modal for exporting a Creator Studio video
|
| 3 |
+
* with in-browser Piper TTS narration.
|
| 4 |
*
|
| 5 |
* Flow:
|
| 6 |
+
* 1. Target β MP4 (edit/share) vs MP4 for YouTube.
|
| 7 |
+
* 2. Voiceover β Piper voice + rate + pitch + subtitle burn-in toggle,
|
| 8 |
+
* with a Preview button.
|
| 9 |
+
* 3. Generate β synthesize narration for every scene that has text
|
| 10 |
+
* (or re-synthesize all if the user checks "regenerate")
|
| 11 |
+
* and upload each WAV to the backend.
|
| 12 |
+
* 4. Render β POST /studio/videos/{id}/export/mp4, poll, download.
|
| 13 |
*
|
| 14 |
* Race protection:
|
| 15 |
+
* Every async op captures a per-mount epoch; closing the modal or
|
| 16 |
+
* starting a new render bumps the epoch and stale responses are
|
| 17 |
+
* dropped. Same pattern as App.tsx's account-switch guard.
|
|
|
|
| 18 |
*/
|
| 19 |
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
| 20 |
+
import {
|
| 21 |
+
AlertTriangle,
|
| 22 |
+
CheckCircle2,
|
| 23 |
+
Download,
|
| 24 |
+
Film,
|
| 25 |
+
Loader2,
|
| 26 |
+
Mic,
|
| 27 |
+
Play,
|
| 28 |
+
Square,
|
| 29 |
+
X,
|
| 30 |
+
Youtube,
|
| 31 |
+
} from 'lucide-react'
|
| 32 |
+
|
| 33 |
+
import {
|
| 34 |
+
listVoices,
|
| 35 |
+
synthesizeToBlob,
|
| 36 |
+
speak as piperSpeak,
|
| 37 |
+
stop as piperStop,
|
| 38 |
+
isSupported as piperSupported,
|
| 39 |
+
getSelectedVoiceId,
|
| 40 |
+
setSelectedVoiceId,
|
| 41 |
+
} from '../tts/piperTts'
|
| 42 |
+
import { DEFAULT_PIPER_VOICE_ID } from '../tts/piperVoices'
|
| 43 |
|
| 44 |
export type ExportKind = 'mp4_plain' | 'mp4_youtube'
|
| 45 |
|
| 46 |
+
/** Minimum scene info the wizard needs. Matches the flat shape used
|
| 47 |
+
* inside CreatorStudioEditor. */
|
| 48 |
+
export interface WizardScene {
|
| 49 |
+
id: string
|
| 50 |
+
idx: number
|
| 51 |
+
narration: string
|
| 52 |
+
audioUrl: string | null
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
interface Props {
|
| 56 |
open: boolean
|
| 57 |
onClose: () => void
|
| 58 |
backendUrl: string
|
| 59 |
videoId: string
|
| 60 |
+
/** Ordered scenes the render will concatenate. */
|
| 61 |
+
scenes: WizardScene[]
|
| 62 |
+
/** Display name shown in the modal header. */
|
| 63 |
videoTitle?: string
|
| 64 |
/** Disable submit when there are no scenes or scenes are still generating. */
|
| 65 |
disabledReason?: string | null
|
|
|
|
| 77 |
|
| 78 |
const POLL_INTERVAL_MS = 1000
|
| 79 |
const POLL_TIMEOUT_MS = 10 * 60 * 1000 // 10 minutes
|
| 80 |
+
const PREVIEW_TEXT = 'Hello, this is a preview of your selected voice.'
|
| 81 |
|
| 82 |
function authHeaders(): Record<string, string> {
|
| 83 |
const token = localStorage.getItem('homepilot_auth_token') || ''
|
|
|
|
| 88 |
return u.replace(/\/+$/, '')
|
| 89 |
}
|
| 90 |
|
| 91 |
+
type Step = 1 | 2 | 3 | 4
|
| 92 |
+
|
| 93 |
export default function CreatorExportWizard({
|
| 94 |
open,
|
| 95 |
onClose,
|
| 96 |
backendUrl,
|
| 97 |
videoId,
|
| 98 |
+
scenes,
|
| 99 |
videoTitle,
|
| 100 |
disabledReason,
|
| 101 |
}: Props) {
|
| 102 |
+
// ββ Step βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 103 |
+
const [step, setStep] = useState<Step>(1)
|
| 104 |
+
|
| 105 |
+
// ββ Step 1 state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 106 |
const [kind, setKind] = useState<ExportKind>('mp4_youtube')
|
| 107 |
+
|
| 108 |
+
// ββ Step 2 state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 109 |
+
const voices = useMemo(() => listVoices(), [])
|
| 110 |
+
const [voiceId, setVoiceId] = useState<string>(() =>
|
| 111 |
+
piperSupported() ? getSelectedVoiceId() : DEFAULT_PIPER_VOICE_ID,
|
| 112 |
+
)
|
| 113 |
+
const [rate, setRate] = useState<number>(0.9)
|
| 114 |
+
const [pitch, setPitch] = useState<number>(1.0)
|
| 115 |
+
const [subtitlesBurnIn, setSubtitlesBurnIn] = useState<boolean>(true)
|
| 116 |
+
// Default: synthesize every scene that has narration text, even if an
|
| 117 |
+
// audioUrl already exists. The user's invariant is "audio from all
|
| 118 |
+
// scenes" at export time, so freshness beats speed by default.
|
| 119 |
+
// "Keep existing audio" below is the opt-in for re-exports of an
|
| 120 |
+
// unchanged project, where regenerating every take is wasted work.
|
| 121 |
+
const [keepExisting, setKeepExisting] = useState<boolean>(false)
|
| 122 |
+
const [previewing, setPreviewing] = useState<boolean>(false)
|
| 123 |
+
const [previewError, setPreviewError] = useState<string | null>(null)
|
| 124 |
+
|
| 125 |
+
// ββ Step 3 state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 126 |
+
const [genIdx, setGenIdx] = useState<number>(0)
|
| 127 |
+
const [genTotal, setGenTotal] = useState<number>(0)
|
| 128 |
+
const [genError, setGenError] = useState<string | null>(null)
|
| 129 |
+
|
| 130 |
+
// ββ Step 4 state βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 131 |
const [job, setJob] = useState<JobView | null>(null)
|
| 132 |
const [submitError, setSubmitError] = useState<string | null>(null)
|
| 133 |
|
| 134 |
+
// Race protection.
|
|
|
|
| 135 |
const epochRef = useRef(0)
|
| 136 |
|
| 137 |
+
// Reset when modal opens or closes.
|
| 138 |
useEffect(() => {
|
| 139 |
if (open) {
|
| 140 |
setStep(1)
|
| 141 |
setJob(null)
|
| 142 |
setSubmitError(null)
|
| 143 |
+
setGenIdx(0)
|
| 144 |
+
setGenTotal(0)
|
| 145 |
+
setGenError(null)
|
| 146 |
+
setPreviewing(false)
|
| 147 |
+
setPreviewError(null)
|
| 148 |
} else {
|
|
|
|
| 149 |
epochRef.current += 1
|
| 150 |
+
try { piperStop() } catch { /* ignore */ }
|
| 151 |
}
|
| 152 |
}, [open])
|
| 153 |
|
| 154 |
+
// Persist voice selection so subsequent renders remember the pick.
|
| 155 |
+
useEffect(() => {
|
| 156 |
+
if (voiceId) setSelectedVoiceId(voiceId)
|
| 157 |
+
}, [voiceId])
|
| 158 |
+
|
| 159 |
const downloadUrl = useMemo(() => {
|
| 160 |
if (!job || job.status !== 'done') return ''
|
| 161 |
const tok = localStorage.getItem('homepilot_auth_token') || ''
|
|
|
|
| 163 |
return `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${job.id}/download${qp}`
|
| 164 |
}, [job, backendUrl, videoId])
|
| 165 |
|
| 166 |
+
// ββ Step 2 β Preview βββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 167 |
+
|
| 168 |
+
const handlePreview = useCallback(async () => {
|
| 169 |
+
setPreviewError(null)
|
| 170 |
+
if (previewing) {
|
| 171 |
+
try { piperStop() } catch { /* ignore */ }
|
| 172 |
+
setPreviewing(false)
|
| 173 |
+
return
|
| 174 |
+
}
|
| 175 |
+
if (!piperSupported()) {
|
| 176 |
+
setPreviewError('Piper requires a secure context (https) with Web Audio + OPFS support.')
|
| 177 |
+
return
|
| 178 |
+
}
|
| 179 |
+
setPreviewing(true)
|
| 180 |
+
try {
|
| 181 |
+
await piperSpeak(PREVIEW_TEXT, {
|
| 182 |
+
voiceId,
|
| 183 |
+
rate,
|
| 184 |
+
onEnd: () => setPreviewing(false),
|
| 185 |
+
onError: (err) => {
|
| 186 |
+
setPreviewError(String(err?.message || err))
|
| 187 |
+
setPreviewing(false)
|
| 188 |
+
},
|
| 189 |
+
})
|
| 190 |
+
} catch (err: any) {
|
| 191 |
+
setPreviewError(String(err?.message || err))
|
| 192 |
+
setPreviewing(false)
|
| 193 |
+
}
|
| 194 |
+
}, [previewing, voiceId, rate])
|
| 195 |
+
|
| 196 |
+
// ββ Step 3 β Generate all scene narrations βββββββββββββββββββββββββββββββ
|
| 197 |
+
|
| 198 |
+
/** Scenes that need synthesis under the current toggle state.
|
| 199 |
+
* Default ("keep existing" off): every scene with narration text.
|
| 200 |
+
* Opt-in ("keep existing" on): only scenes that do NOT already
|
| 201 |
+
* have an uploaded audioUrl. */
|
| 202 |
+
const scenesToSynthesize = useMemo(
|
| 203 |
+
() =>
|
| 204 |
+
scenes.filter((s) => {
|
| 205 |
+
const hasText = (s.narration || '').trim().length > 0
|
| 206 |
+
if (!hasText) return false
|
| 207 |
+
if (keepExisting && s.audioUrl) return false
|
| 208 |
+
return true
|
| 209 |
+
}),
|
| 210 |
+
[scenes, keepExisting],
|
| 211 |
+
)
|
| 212 |
+
|
| 213 |
+
const handleGenerate = useCallback(async () => {
|
| 214 |
+
if (!piperSupported()) {
|
| 215 |
+
setGenError('Piper is not supported in this browser. Use a Chromium-based browser on https.')
|
| 216 |
+
return
|
| 217 |
+
}
|
| 218 |
+
epochRef.current += 1
|
| 219 |
+
const myEpoch = epochRef.current
|
| 220 |
+
setStep(3)
|
| 221 |
+
setGenError(null)
|
| 222 |
+
setGenIdx(0)
|
| 223 |
+
setGenTotal(scenesToSynthesize.length)
|
| 224 |
+
|
| 225 |
+
if (scenesToSynthesize.length === 0) {
|
| 226 |
+
// Nothing to synthesize β jump straight to render.
|
| 227 |
+
setStep(4)
|
| 228 |
+
void startRender(myEpoch)
|
| 229 |
+
return
|
| 230 |
+
}
|
| 231 |
+
|
| 232 |
+
for (let i = 0; i < scenesToSynthesize.length; i++) {
|
| 233 |
+
if (epochRef.current !== myEpoch) return
|
| 234 |
+
const scene = scenesToSynthesize[i]
|
| 235 |
+
try {
|
| 236 |
+
const blob = await synthesizeToBlob(scene.narration, { voiceId })
|
| 237 |
+
if (epochRef.current !== myEpoch) return
|
| 238 |
+
|
| 239 |
+
// Upload to the backend so render_mp4 can mix it in.
|
| 240 |
+
const fd = new FormData()
|
| 241 |
+
fd.append('file', blob, `${scene.id}.wav`)
|
| 242 |
+
const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/scenes/${scene.id}/narration`
|
| 243 |
+
const r = await fetch(url, {
|
| 244 |
+
method: 'POST',
|
| 245 |
+
headers: { ...authHeaders() },
|
| 246 |
+
body: fd,
|
| 247 |
+
})
|
| 248 |
+
if (epochRef.current !== myEpoch) return
|
| 249 |
+
if (!r.ok) {
|
| 250 |
+
const text = await r.text().catch(() => '')
|
| 251 |
+
throw new Error(text || `Upload failed (HTTP ${r.status})`)
|
| 252 |
+
}
|
| 253 |
+
} catch (err: any) {
|
| 254 |
+
if (epochRef.current !== myEpoch) return
|
| 255 |
+
setGenError(
|
| 256 |
+
`Scene ${scene.idx + 1}: ${String(err?.message || err)}`,
|
| 257 |
+
)
|
| 258 |
+
return
|
| 259 |
+
}
|
| 260 |
+
setGenIdx(i + 1)
|
| 261 |
+
}
|
| 262 |
+
|
| 263 |
+
if (epochRef.current !== myEpoch) return
|
| 264 |
+
// All done β kick off the render.
|
| 265 |
+
setStep(4)
|
| 266 |
+
void startRender(myEpoch)
|
| 267 |
+
}, [scenesToSynthesize, voiceId, backendUrl, videoId])
|
| 268 |
+
|
| 269 |
+
// ββ Step 4 β Render + poll βββββββββββββββββββββββββββββββββββββββββββββββ
|
| 270 |
+
|
| 271 |
const pollJob = useCallback(
|
| 272 |
async (jobId: string, myEpoch: number) => {
|
| 273 |
const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${jobId}`
|
| 274 |
const start = Date.now()
|
|
|
|
|
|
|
| 275 |
while (Date.now() - start < POLL_TIMEOUT_MS) {
|
| 276 |
if (epochRef.current !== myEpoch) return
|
| 277 |
try {
|
|
|
|
| 304 |
}
|
| 305 |
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
|
| 306 |
}
|
|
|
|
|
|
|
| 307 |
if (epochRef.current === myEpoch) {
|
| 308 |
setJob((prev) =>
|
| 309 |
prev ? { ...prev, status: 'error', error: 'Render timed out' } : prev,
|
|
|
|
| 313 |
[backendUrl, videoId],
|
| 314 |
)
|
| 315 |
|
| 316 |
+
const startRender = useCallback(
|
| 317 |
+
async (myEpoch: number) => {
|
| 318 |
+
setSubmitError(null)
|
| 319 |
+
setJob({ id: '', status: 'queued', progress: 0, kind })
|
| 320 |
+
try {
|
| 321 |
+
const r = await fetch(
|
| 322 |
+
`${trimBackend(backendUrl)}/studio/videos/${videoId}/export/mp4`,
|
| 323 |
+
{
|
| 324 |
+
method: 'POST',
|
| 325 |
+
headers: { 'Content-Type': 'application/json', ...authHeaders() },
|
| 326 |
+
body: JSON.stringify({
|
| 327 |
+
kind,
|
| 328 |
+
audio_rate: rate,
|
| 329 |
+
audio_pitch: pitch,
|
| 330 |
+
subtitles: subtitlesBurnIn ? 'burn_in' : 'none',
|
| 331 |
+
}),
|
| 332 |
+
},
|
| 333 |
+
)
|
| 334 |
+
if (epochRef.current !== myEpoch) return
|
| 335 |
+
if (!r.ok) {
|
| 336 |
+
const text = await r.text().catch(() => '')
|
| 337 |
+
const detail = text || `HTTP ${r.status}`
|
| 338 |
+
setSubmitError(detail)
|
| 339 |
+
setJob({ id: '', status: 'error', progress: 0, error: detail, kind })
|
| 340 |
+
return
|
| 341 |
+
}
|
| 342 |
+
const j = await r.json()
|
| 343 |
+
if (epochRef.current !== myEpoch) return
|
| 344 |
+
setJob({
|
| 345 |
+
id: j.id,
|
| 346 |
+
status: j.status,
|
| 347 |
+
progress: typeof j.progress === 'number' ? j.progress : 0,
|
| 348 |
+
kind: j.kind,
|
| 349 |
+
})
|
| 350 |
+
void pollJob(j.id, myEpoch)
|
| 351 |
+
} catch (err: any) {
|
| 352 |
+
if (epochRef.current !== myEpoch) return
|
| 353 |
+
const msg = String(err?.message || err)
|
| 354 |
+
setSubmitError(msg)
|
| 355 |
+
setJob({ id: '', status: 'error', progress: 0, error: msg, kind })
|
| 356 |
}
|
| 357 |
+
},
|
| 358 |
+
[backendUrl, videoId, kind, rate, pitch, subtitlesBurnIn, pollJob],
|
| 359 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
|
| 361 |
if (!open) return null
|
| 362 |
|
| 363 |
const submitDisabled = Boolean(disabledReason)
|
| 364 |
|
| 365 |
+
// ββ UI βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 366 |
+
|
| 367 |
return (
|
| 368 |
<div
|
| 369 |
className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
|
|
|
|
| 373 |
aria-label="Export video"
|
| 374 |
>
|
| 375 |
<div
|
| 376 |
+
className="w-full max-w-xl rounded-2xl border border-white/10 bg-[#0b0b0c] text-white shadow-2xl max-h-[90vh] overflow-hidden flex flex-col"
|
| 377 |
onClick={(e) => e.stopPropagation()}
|
| 378 |
>
|
| 379 |
+
{/* Header */}
|
| 380 |
+
<div className="flex items-center justify-between border-b border-white/10 px-5 py-3 flex-shrink-0">
|
| 381 |
<div>
|
| 382 |
<div className="text-sm font-semibold">Export video</div>
|
| 383 |
{videoTitle ? (
|
| 384 |
<div className="text-xs text-white/50 truncate max-w-[28rem]">{videoTitle}</div>
|
| 385 |
) : null}
|
| 386 |
</div>
|
| 387 |
+
<div className="flex items-center gap-3 text-[11px] text-white/40">
|
| 388 |
+
<span className={step >= 1 ? 'text-white/80' : ''}>1 Β· Target</span>
|
| 389 |
+
<span>βΊ</span>
|
| 390 |
+
<span className={step >= 2 ? 'text-white/80' : ''}>2 Β· Voiceover</span>
|
| 391 |
+
<span>βΊ</span>
|
| 392 |
+
<span className={step >= 3 ? 'text-white/80' : ''}>3 Β· Narration</span>
|
| 393 |
+
<span>βΊ</span>
|
| 394 |
+
<span className={step >= 4 ? 'text-white/80' : ''}>4 Β· Render</span>
|
| 395 |
+
</div>
|
| 396 |
<button
|
| 397 |
type="button"
|
| 398 |
onClick={onClose}
|
|
|
|
| 403 |
</button>
|
| 404 |
</div>
|
| 405 |
|
| 406 |
+
{/* Body */}
|
| 407 |
+
<div className="px-5 py-5 space-y-4 overflow-y-auto">
|
| 408 |
{step === 1 && (
|
| 409 |
<>
|
| 410 |
<div className="text-xs text-white/60">
|
|
|
|
| 478 |
</button>
|
| 479 |
<button
|
| 480 |
type="button"
|
| 481 |
+
onClick={() => setStep(2)}
|
| 482 |
disabled={submitDisabled}
|
| 483 |
className={[
|
| 484 |
'px-3 py-1.5 rounded-lg text-sm font-medium',
|
|
|
|
| 487 |
: 'bg-white text-black hover:bg-white/90',
|
| 488 |
].join(' ')}
|
| 489 |
>
|
| 490 |
+
Next: Voiceover
|
| 491 |
</button>
|
| 492 |
</div>
|
| 493 |
</>
|
| 494 |
)}
|
| 495 |
|
| 496 |
{step === 2 && (
|
| 497 |
+
<>
|
| 498 |
+
<div className="flex items-center gap-2 text-xs text-white/60">
|
| 499 |
+
<Mic size={14} className="text-cyan-300" />
|
| 500 |
+
Piper runs fully in-browser via WebAssembly. First use
|
| 501 |
+
downloads a voice model (~20 MB, cached).
|
| 502 |
+
</div>
|
| 503 |
+
|
| 504 |
+
<div className="space-y-3">
|
| 505 |
+
<label className="block text-[11px] uppercase tracking-wide text-white/50">
|
| 506 |
+
Piper voice
|
| 507 |
+
</label>
|
| 508 |
+
<select
|
| 509 |
+
value={voiceId}
|
| 510 |
+
onChange={(e) => setVoiceId(e.target.value)}
|
| 511 |
+
className="w-full px-3 py-2 rounded-lg bg-white/5 border border-white/10 text-sm text-white/90 focus:outline-none focus:border-white/30"
|
| 512 |
+
>
|
| 513 |
+
{voices.map((v) => (
|
| 514 |
+
<option key={v.id} value={v.id}>
|
| 515 |
+
{v.name} β {v.lang} Β· {v.gender} Β· {v.quality}
|
| 516 |
+
</option>
|
| 517 |
+
))}
|
| 518 |
+
</select>
|
| 519 |
+
</div>
|
| 520 |
+
|
| 521 |
+
<div className="grid grid-cols-2 gap-4">
|
| 522 |
+
<div>
|
| 523 |
+
<div className="flex items-center justify-between text-[11px] uppercase tracking-wide text-white/50">
|
| 524 |
+
<span>Rate</span>
|
| 525 |
+
<span className="font-mono text-white/70">{rate.toFixed(2)}</span>
|
| 526 |
+
</div>
|
| 527 |
+
<input
|
| 528 |
+
type="range"
|
| 529 |
+
min={0.5}
|
| 530 |
+
max={2.0}
|
| 531 |
+
step={0.05}
|
| 532 |
+
value={rate}
|
| 533 |
+
onChange={(e) => setRate(Number(e.target.value))}
|
| 534 |
+
className="w-full accent-cyan-400"
|
| 535 |
+
/>
|
| 536 |
+
</div>
|
| 537 |
+
<div>
|
| 538 |
+
<div className="flex items-center justify-between text-[11px] uppercase tracking-wide text-white/50">
|
| 539 |
+
<span>Pitch</span>
|
| 540 |
+
<span className="font-mono text-white/70">{pitch.toFixed(2)}</span>
|
| 541 |
+
</div>
|
| 542 |
+
<input
|
| 543 |
+
type="range"
|
| 544 |
+
min={0.5}
|
| 545 |
+
max={2.0}
|
| 546 |
+
step={0.05}
|
| 547 |
+
value={pitch}
|
| 548 |
+
onChange={(e) => setPitch(Number(e.target.value))}
|
| 549 |
+
className="w-full accent-cyan-400"
|
| 550 |
+
/>
|
| 551 |
+
</div>
|
| 552 |
+
</div>
|
| 553 |
+
|
| 554 |
+
<div className="flex items-center gap-2">
|
| 555 |
+
<button
|
| 556 |
+
type="button"
|
| 557 |
+
onClick={handlePreview}
|
| 558 |
+
className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15 border border-white/10"
|
| 559 |
+
>
|
| 560 |
+
{previewing ? <Square size={14} /> : <Play size={14} />}
|
| 561 |
+
{previewing ? 'Stop preview' : 'Preview voice'}
|
| 562 |
+
</button>
|
| 563 |
+
<span className="text-[11px] text-white/40 truncate">{PREVIEW_TEXT}</span>
|
| 564 |
+
</div>
|
| 565 |
+
|
| 566 |
+
{previewError ? (
|
| 567 |
+
<div className="text-[11px] text-red-300/80">{previewError}</div>
|
| 568 |
+
) : null}
|
| 569 |
+
|
| 570 |
+
<label className="flex items-center gap-2 text-sm text-white/80 cursor-pointer">
|
| 571 |
+
<input
|
| 572 |
+
type="checkbox"
|
| 573 |
+
checked={subtitlesBurnIn}
|
| 574 |
+
onChange={(e) => setSubtitlesBurnIn(e.target.checked)}
|
| 575 |
+
className="accent-cyan-400"
|
| 576 |
+
/>
|
| 577 |
+
Burn subtitles into the video (from scene narration text)
|
| 578 |
+
</label>
|
| 579 |
+
|
| 580 |
+
<label className="flex items-center gap-2 text-sm text-white/80 cursor-pointer">
|
| 581 |
+
<input
|
| 582 |
+
type="checkbox"
|
| 583 |
+
checked={keepExisting}
|
| 584 |
+
onChange={(e) => setKeepExisting(e.target.checked)}
|
| 585 |
+
className="accent-cyan-400"
|
| 586 |
+
/>
|
| 587 |
+
Keep existing narration audio where present
|
| 588 |
+
<span className="text-[11px] text-white/40">(skip scenes that already have a take)</span>
|
| 589 |
+
</label>
|
| 590 |
+
|
| 591 |
+
<div className="text-[11px] text-white/40">
|
| 592 |
+
{scenesToSynthesize.length === 0
|
| 593 |
+
? 'No scenes have narration text β the export will have silent audio.'
|
| 594 |
+
: `${scenesToSynthesize.length} of ${scenes.length} scene${scenes.length === 1 ? '' : 's'} will be synthesized with the selected voice.`}
|
| 595 |
+
</div>
|
| 596 |
+
|
| 597 |
+
<div className="flex justify-between gap-2 pt-2">
|
| 598 |
+
<button
|
| 599 |
+
type="button"
|
| 600 |
+
onClick={() => setStep(1)}
|
| 601 |
+
className="px-3 py-1.5 rounded-lg text-sm text-white/70 hover:bg-white/10"
|
| 602 |
+
>
|
| 603 |
+
Back
|
| 604 |
+
</button>
|
| 605 |
+
<button
|
| 606 |
+
type="button"
|
| 607 |
+
onClick={handleGenerate}
|
| 608 |
+
className="px-3 py-1.5 rounded-lg text-sm font-medium bg-white text-black hover:bg-white/90"
|
| 609 |
+
>
|
| 610 |
+
{scenesToSynthesize.length === 0 ? 'Skip & render' : 'Generate narration'}
|
| 611 |
+
</button>
|
| 612 |
+
</div>
|
| 613 |
+
</>
|
| 614 |
+
)}
|
| 615 |
+
|
| 616 |
+
{step === 3 && (
|
| 617 |
+
<>
|
| 618 |
+
<div className="flex items-center gap-2 text-white/80">
|
| 619 |
+
<Loader2 size={16} className="animate-spin" />
|
| 620 |
+
<span className="text-sm">Synthesizing narrationβ¦</span>
|
| 621 |
+
</div>
|
| 622 |
+
<div className="h-2 rounded-full bg-white/10 overflow-hidden">
|
| 623 |
+
<div
|
| 624 |
+
className="h-full bg-cyan-400 transition-all"
|
| 625 |
+
style={{
|
| 626 |
+
width: `${genTotal === 0 ? 0 : Math.round((genIdx / genTotal) * 100)}%`,
|
| 627 |
+
}}
|
| 628 |
+
/>
|
| 629 |
+
</div>
|
| 630 |
+
<div className="text-[11px] text-white/40 text-center">
|
| 631 |
+
{genIdx} of {genTotal} scenes
|
| 632 |
+
</div>
|
| 633 |
+
|
| 634 |
+
{genError ? (
|
| 635 |
+
<div className="space-y-3">
|
| 636 |
+
<div className="flex items-start gap-2 rounded-lg border border-red-400/30 bg-red-400/10 p-3 text-[12px] text-red-200">
|
| 637 |
+
<AlertTriangle size={14} className="mt-0.5 flex-shrink-0" />
|
| 638 |
+
<div>
|
| 639 |
+
<div className="font-medium mb-0.5">Narration synthesis failed</div>
|
| 640 |
+
<div className="text-red-300/80 break-words">{genError}</div>
|
| 641 |
+
</div>
|
| 642 |
+
</div>
|
| 643 |
+
<button
|
| 644 |
+
type="button"
|
| 645 |
+
onClick={() => setStep(2)}
|
| 646 |
+
className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
|
| 647 |
+
>
|
| 648 |
+
Back to voiceover
|
| 649 |
+
</button>
|
| 650 |
+
</div>
|
| 651 |
+
) : null}
|
| 652 |
+
</>
|
| 653 |
+
)}
|
| 654 |
+
|
| 655 |
+
{step === 4 && (
|
| 656 |
<>
|
| 657 |
{job?.status === 'done' ? (
|
| 658 |
<div className="space-y-4">
|
|
|
|
| 670 |
</a>
|
| 671 |
<div className="text-[11px] text-white/40 text-center">
|
| 672 |
{job.kind === 'mp4_youtube' ? 'YouTube-optimized' : 'Plain MP4'}
|
| 673 |
+
{subtitlesBurnIn ? ' Β· subtitles burned in' : ''}
|
| 674 |
</div>
|
| 675 |
</div>
|
| 676 |
) : job?.status === 'error' ? (
|
|
|
|
| 686 |
</div>
|
| 687 |
<button
|
| 688 |
type="button"
|
| 689 |
+
onClick={() => setStep(2)}
|
| 690 |
className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
|
| 691 |
>
|
| 692 |
+
Back to voiceover
|
| 693 |
</button>
|
| 694 |
</div>
|
| 695 |
) : (
|
|
|
|
| 703 |
<div className="h-2 rounded-full bg-white/10 overflow-hidden">
|
| 704 |
<div
|
| 705 |
className="h-full bg-cyan-400 transition-all"
|
| 706 |
+
style={{
|
| 707 |
+
width: `${Math.max(2, Math.min(100, job?.progress || 0))}%`,
|
| 708 |
+
}}
|
| 709 |
/>
|
| 710 |
</div>
|
| 711 |
<div className="text-[11px] text-white/40 text-center">
|
frontend/src/ui/components/TtsEngineSection.tsx
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Schema-driven Settings section for the active TTS engine.
|
| 3 |
+
*
|
| 4 |
+
* This component is deliberately self-contained so SettingsPanel can mount
|
| 5 |
+
* it with one line and stay agnostic of which engines are installed.
|
| 6 |
+
*
|
| 7 |
+
* Behavior:
|
| 8 |
+
* - Dropdown to pick the active engine (only ``isAvailable()`` ones).
|
| 9 |
+
* - Below: controls rendered from the active provider's
|
| 10 |
+
* ``getSettingsSchema()`` β ``select`` / ``range`` / ``toggle``.
|
| 11 |
+
* - The Web Speech provider's ``voiceId`` schema has a placeholder
|
| 12 |
+
* options list; we merge in the browser's live voices at render
|
| 13 |
+
* time so the user sees their actual installed voices.
|
| 14 |
+
* - Every edit persists to the registry's per-user scoped settings
|
| 15 |
+
* bucket AND to ``value`` via ``onChangeDraft`` so the existing
|
| 16 |
+
* Voice Assistant code continues to read ``value.selectedVoice``.
|
| 17 |
+
*
|
| 18 |
+
* Purely additive: the existing System Voice select stays untouched
|
| 19 |
+
* above this component, and this section only appears when the user
|
| 20 |
+
* opts into it by picking a non-default engine (or leaves it on
|
| 21 |
+
* ``web-speech-api`` where we render the same controls the old UI did).
|
| 22 |
+
*/
|
| 23 |
+
|
| 24 |
+
import React, { useEffect, useMemo, useState } from 'react'
|
| 25 |
+
import {
|
| 26 |
+
getActiveTtsEngineId,
|
| 27 |
+
listTtsProviders,
|
| 28 |
+
onActiveTtsEngineChange,
|
| 29 |
+
readTtsProviderSettings,
|
| 30 |
+
setActiveTtsEngine,
|
| 31 |
+
writeTtsProviderSettings,
|
| 32 |
+
} from '../tts'
|
| 33 |
+
import type { SettingsField, TtsProvider } from '../tts'
|
| 34 |
+
|
| 35 |
+
interface Props {
|
| 36 |
+
/** Optional: used by the Web Speech engine to populate its voice
|
| 37 |
+
* dropdown with the browser's live voices. Passing undefined is
|
| 38 |
+
* safe β we fall back to ``getVoices()``. */
|
| 39 |
+
systemVoices?: readonly SpeechSynthesisVoice[]
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
function _fieldValue(schema: SettingsField, saved: Record<string, unknown>): string | number | boolean {
|
| 43 |
+
const v = saved[schema.key]
|
| 44 |
+
if (schema.kind === 'range') {
|
| 45 |
+
return typeof v === 'number' ? v : schema.defaultValue
|
| 46 |
+
}
|
| 47 |
+
if (schema.kind === 'toggle') {
|
| 48 |
+
return typeof v === 'boolean' ? v : schema.defaultValue
|
| 49 |
+
}
|
| 50 |
+
return typeof v === 'string' ? v : schema.defaultValue
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
function _mergeWebSpeechOptions(
|
| 54 |
+
field: SettingsField,
|
| 55 |
+
voices: readonly SpeechSynthesisVoice[],
|
| 56 |
+
): SettingsField {
|
| 57 |
+
if (field.kind !== 'select' || field.key !== 'voiceId' || voices.length === 0) {
|
| 58 |
+
return field
|
| 59 |
+
}
|
| 60 |
+
const live = voices.map((v) => ({
|
| 61 |
+
value: v.voiceURI || v.name,
|
| 62 |
+
label: `${v.name} (${v.lang})`,
|
| 63 |
+
}))
|
| 64 |
+
return { ...field, options: [...field.options, ...live] }
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
export default function TtsEngineSection({ systemVoices }: Props): JSX.Element {
|
| 68 |
+
const [activeId, setActiveIdState] = useState<string>(() => getActiveTtsEngineId())
|
| 69 |
+
// Re-render when another component swaps the active engine.
|
| 70 |
+
useEffect(() => onActiveTtsEngineChange((id) => setActiveIdState(id)), [])
|
| 71 |
+
|
| 72 |
+
const providers = useMemo<readonly TtsProvider[]>(
|
| 73 |
+
() => listTtsProviders().filter((p) => p.isAvailable()),
|
| 74 |
+
[activeId],
|
| 75 |
+
)
|
| 76 |
+
|
| 77 |
+
const active = useMemo<TtsProvider | undefined>(
|
| 78 |
+
() => providers.find((p) => p.id === activeId),
|
| 79 |
+
[providers, activeId],
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
// Saved settings blob for the active provider.
|
| 83 |
+
const [settings, setSettings] = useState<Record<string, unknown>>(
|
| 84 |
+
() => readTtsProviderSettings(activeId),
|
| 85 |
+
)
|
| 86 |
+
useEffect(() => {
|
| 87 |
+
setSettings(readTtsProviderSettings(activeId))
|
| 88 |
+
}, [activeId])
|
| 89 |
+
|
| 90 |
+
// When the default engine is selected the host panel already renders
|
| 91 |
+
// a dedicated "Assistant Voice" / "System Voice" dropdown above us
|
| 92 |
+
// that covers voice + rate + pitch for Web Speech. Rendering our
|
| 93 |
+
// schema-driven twin on top of it is pure duplication β skip it
|
| 94 |
+
// unless the user has opted into a non-default engine (Piper etc.).
|
| 95 |
+
const isDefaultEngine = activeId === 'web-speech-api'
|
| 96 |
+
const schema = active && !isDefaultEngine ? active.getSettingsSchema() : []
|
| 97 |
+
|
| 98 |
+
// Test-voice state: the button speaks through whichever engine is
|
| 99 |
+
// active right now so the user can hear their pick without leaving
|
| 100 |
+
// Settings. Mirrors the Preview button in the Creator Studio wizard.
|
| 101 |
+
const [testing, setTesting] = useState(false)
|
| 102 |
+
const [testError, setTestError] = useState<string | null>(null)
|
| 103 |
+
const onTest = () => {
|
| 104 |
+
setTestError(null)
|
| 105 |
+
if (!active) return
|
| 106 |
+
if (testing) {
|
| 107 |
+
try { active.stop() } catch { /* ignore */ }
|
| 108 |
+
setTesting(false)
|
| 109 |
+
return
|
| 110 |
+
}
|
| 111 |
+
setTesting(true)
|
| 112 |
+
const voiceId = typeof settings.voiceId === 'string' ? settings.voiceId : undefined
|
| 113 |
+
const rate = typeof settings.rate === 'number' ? settings.rate : undefined
|
| 114 |
+
const pitch = typeof settings.pitch === 'number' ? settings.pitch : undefined
|
| 115 |
+
active
|
| 116 |
+
.speak('Hello, this is a preview of your selected voice.', {
|
| 117 |
+
voiceId,
|
| 118 |
+
rate,
|
| 119 |
+
pitch,
|
| 120 |
+
onEnd: () => setTesting(false),
|
| 121 |
+
onError: (err) => {
|
| 122 |
+
setTestError(String(err?.message || err))
|
| 123 |
+
setTesting(false)
|
| 124 |
+
},
|
| 125 |
+
})
|
| 126 |
+
.catch((err) => {
|
| 127 |
+
setTestError(String(err?.message || err))
|
| 128 |
+
setTesting(false)
|
| 129 |
+
})
|
| 130 |
+
}
|
| 131 |
+
|
| 132 |
+
const onChangeField = (key: string, value: string | number | boolean) => {
|
| 133 |
+
const next = { ...settings, [key]: value }
|
| 134 |
+
setSettings(next)
|
| 135 |
+
writeTtsProviderSettings(activeId, { [key]: value })
|
| 136 |
+
}
|
| 137 |
+
|
| 138 |
+
return (
|
| 139 |
+
<div className="border-t border-white/5 pt-3">
|
| 140 |
+
<div className="text-[11px] uppercase tracking-wider text-white/40 mb-2 font-semibold">
|
| 141 |
+
TTS Engine
|
| 142 |
+
</div>
|
| 143 |
+
|
| 144 |
+
<div className="space-y-3">
|
| 145 |
+
<div>
|
| 146 |
+
<label className="block text-[10px] text-white/50 mb-2">Engine</label>
|
| 147 |
+
<select
|
| 148 |
+
className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
|
| 149 |
+
value={activeId}
|
| 150 |
+
onChange={(e) => setActiveTtsEngine(e.target.value)}
|
| 151 |
+
>
|
| 152 |
+
{providers.map((p) => (
|
| 153 |
+
<option key={p.id} value={p.id}>
|
| 154 |
+
{p.displayName}
|
| 155 |
+
</option>
|
| 156 |
+
))}
|
| 157 |
+
</select>
|
| 158 |
+
{active?.id === 'piper-wasm' && (
|
| 159 |
+
<div className="mt-1 text-[10px] text-white/40 leading-relaxed">
|
| 160 |
+
Piper runs fully in-browser via WebAssembly. First use
|
| 161 |
+
downloads a voice model (~20 MB, cached). Falls back to the
|
| 162 |
+
HomePilot mirror when the upstream CDN is unreachable.
|
| 163 |
+
</div>
|
| 164 |
+
)}
|
| 165 |
+
</div>
|
| 166 |
+
|
| 167 |
+
{active && !isDefaultEngine ? (
|
| 168 |
+
schema.map((field) => {
|
| 169 |
+
const resolved =
|
| 170 |
+
active.id === 'web-speech-api' && field.kind === 'select' && field.key === 'voiceId'
|
| 171 |
+
? _mergeWebSpeechOptions(
|
| 172 |
+
field,
|
| 173 |
+
systemVoices ??
|
| 174 |
+
(typeof window !== 'undefined' && 'speechSynthesis' in window
|
| 175 |
+
? window.speechSynthesis.getVoices()
|
| 176 |
+
: []),
|
| 177 |
+
)
|
| 178 |
+
: field
|
| 179 |
+
const value = _fieldValue(resolved, settings)
|
| 180 |
+
|
| 181 |
+
if (resolved.kind === 'select') {
|
| 182 |
+
return (
|
| 183 |
+
<div key={resolved.key}>
|
| 184 |
+
<label className="block text-[10px] text-white/50 mb-2">
|
| 185 |
+
{resolved.label}
|
| 186 |
+
</label>
|
| 187 |
+
<select
|
| 188 |
+
className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
|
| 189 |
+
value={String(value)}
|
| 190 |
+
onChange={(e) => onChangeField(resolved.key, e.target.value)}
|
| 191 |
+
>
|
| 192 |
+
{resolved.options.map((o) => (
|
| 193 |
+
<option key={o.value} value={o.value}>
|
| 194 |
+
{o.label}
|
| 195 |
+
</option>
|
| 196 |
+
))}
|
| 197 |
+
</select>
|
| 198 |
+
{resolved.description ? (
|
| 199 |
+
<div className="mt-1 text-[10px] text-white/40">{resolved.description}</div>
|
| 200 |
+
) : null}
|
| 201 |
+
</div>
|
| 202 |
+
)
|
| 203 |
+
}
|
| 204 |
+
|
| 205 |
+
if (resolved.kind === 'range') {
|
| 206 |
+
return (
|
| 207 |
+
<div key={resolved.key}>
|
| 208 |
+
<div className="flex items-center justify-between">
|
| 209 |
+
<label className="text-[10px] text-white/50">{resolved.label}</label>
|
| 210 |
+
<span className="text-[10px] text-white/60 font-mono">
|
| 211 |
+
{typeof value === 'number' ? value.toFixed(2) : value}
|
| 212 |
+
</span>
|
| 213 |
+
</div>
|
| 214 |
+
<input
|
| 215 |
+
type="range"
|
| 216 |
+
min={resolved.min}
|
| 217 |
+
max={resolved.max}
|
| 218 |
+
step={resolved.step}
|
| 219 |
+
value={typeof value === 'number' ? value : resolved.defaultValue}
|
| 220 |
+
onChange={(e) => onChangeField(resolved.key, Number(e.target.value))}
|
| 221 |
+
className="w-full accent-cyan-400"
|
| 222 |
+
/>
|
| 223 |
+
</div>
|
| 224 |
+
)
|
| 225 |
+
}
|
| 226 |
+
|
| 227 |
+
// toggle
|
| 228 |
+
return (
|
| 229 |
+
<label key={resolved.key} className="flex items-center gap-2 cursor-pointer">
|
| 230 |
+
<input
|
| 231 |
+
type="checkbox"
|
| 232 |
+
checked={Boolean(value)}
|
| 233 |
+
onChange={(e) => onChangeField(resolved.key, e.target.checked)}
|
| 234 |
+
className="w-4 h-4 rounded"
|
| 235 |
+
/>
|
| 236 |
+
<span className="text-xs text-white">{resolved.label}</span>
|
| 237 |
+
</label>
|
| 238 |
+
)
|
| 239 |
+
})
|
| 240 |
+
) : !active ? (
|
| 241 |
+
<div className="text-[10px] text-white/40">
|
| 242 |
+
No TTS engine available in this environment.
|
| 243 |
+
</div>
|
| 244 |
+
) : null}
|
| 245 |
+
|
| 246 |
+
{active && !isDefaultEngine && !active.capabilities.pitch && (
|
| 247 |
+
<div className="text-[10px] text-white/40 italic">
|
| 248 |
+
The {active.displayName.split(' (')[0]} engine does not support a pitch control in
|
| 249 |
+
playback. The Creator Studio export applies pitch post-synthesis via ffmpeg.
|
| 250 |
+
</div>
|
| 251 |
+
)}
|
| 252 |
+
|
| 253 |
+
{/* Test voice button β positioned AFTER the voice + rate + pitch
|
| 254 |
+
controls so users pick their voice first, then preview it
|
| 255 |
+
(best-practice: action sits at the end of the configuration
|
| 256 |
+
flow). Always visible; mirrors the "Preview voice" affordance
|
| 257 |
+
in the Creator Studio export wizard. */}
|
| 258 |
+
{active && (
|
| 259 |
+
<div className="flex items-center gap-2 pt-1">
|
| 260 |
+
<button
|
| 261 |
+
type="button"
|
| 262 |
+
onClick={onTest}
|
| 263 |
+
className="text-[11px] px-3 py-1.5 rounded-lg bg-white/10 hover:bg-white/15 border border-white/10 text-white/80"
|
| 264 |
+
aria-pressed={testing}
|
| 265 |
+
>
|
| 266 |
+
{testing ? 'Stop' : 'Test voice'}
|
| 267 |
+
</button>
|
| 268 |
+
<span className="text-[10px] text-white/40">
|
| 269 |
+
Hello, this is a preview of your selected voice.
|
| 270 |
+
</span>
|
| 271 |
+
</div>
|
| 272 |
+
)}
|
| 273 |
+
{testError ? (
|
| 274 |
+
<div className="text-[10px] text-red-300/80">{testError}</div>
|
| 275 |
+
) : null}
|
| 276 |
+
</div>
|
| 277 |
+
</div>
|
| 278 |
+
)
|
| 279 |
+
}
|
frontend/src/ui/lib/backendUrl.test.ts
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { afterEach, describe, expect, it, vi } from 'vitest'
|
| 2 |
+
import { resolveBackendUrl } from './backendUrl'
|
| 3 |
+
|
| 4 |
+
afterEach(() => {
|
| 5 |
+
window.localStorage.clear()
|
| 6 |
+
vi.unstubAllGlobals()
|
| 7 |
+
})
|
| 8 |
+
|
| 9 |
+
describe('resolveBackendUrl', () => {
|
| 10 |
+
it('falls back to localhost:8000 when stored backend points to local vite dev port', () => {
|
| 11 |
+
vi.stubGlobal('location', {
|
| 12 |
+
hostname: 'localhost',
|
| 13 |
+
origin: 'http://localhost:3000',
|
| 14 |
+
port: '3000',
|
| 15 |
+
protocol: 'http:',
|
| 16 |
+
})
|
| 17 |
+
|
| 18 |
+
window.localStorage.setItem('homepilot_backend_url', 'http://localhost:3000')
|
| 19 |
+
|
| 20 |
+
expect(resolveBackendUrl()).toBe('http://localhost:8000')
|
| 21 |
+
})
|
| 22 |
+
|
| 23 |
+
it('keeps stored localhost backend when it targets a non-vite port', () => {
|
| 24 |
+
vi.stubGlobal('location', {
|
| 25 |
+
hostname: 'localhost',
|
| 26 |
+
origin: 'http://localhost:3000',
|
| 27 |
+
port: '3000',
|
| 28 |
+
protocol: 'http:',
|
| 29 |
+
})
|
| 30 |
+
|
| 31 |
+
window.localStorage.setItem('homepilot_backend_url', 'http://localhost:8010')
|
| 32 |
+
|
| 33 |
+
expect(resolveBackendUrl()).toBe('http://localhost:8010')
|
| 34 |
+
})
|
| 35 |
+
})
|
frontend/src/ui/lib/backendUrl.ts
CHANGED
|
@@ -19,7 +19,12 @@
|
|
| 19 |
*/
|
| 20 |
|
| 21 |
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1"]);
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
|
| 24 |
function isLocalHost(hostname: string): boolean {
|
| 25 |
return LOCAL_HOSTS.has(hostname);
|
|
@@ -70,13 +75,25 @@ export function resolveBackendUrl(override?: string | null): string {
|
|
| 70 |
if (stored && stored.trim()) {
|
| 71 |
const clean = stripTrailingSlash(stored.trim());
|
| 72 |
const pageIsRemote = !isLocalHost(window.location.hostname);
|
| 73 |
-
const
|
|
|
|
|
|
|
| 74 |
const storedIsLocal = storedHost !== null && isLocalHost(storedHost);
|
| 75 |
if (pageIsRemote && storedIsLocal) {
|
| 76 |
// Stale setting from a previous local dev session. Ignore it and
|
| 77 |
// let getDefaultBackendUrl() take over (same origin).
|
| 78 |
return getDefaultBackendUrl();
|
| 79 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 80 |
return clean;
|
| 81 |
}
|
| 82 |
} catch {
|
|
|
|
| 19 |
*/
|
| 20 |
|
| 21 |
const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1"]);
|
| 22 |
+
// Vite dev ports used across the repo. The Makefile's `make start` target
|
| 23 |
+
// launches vite with --port 3000; earlier tooling used Vite's defaults
|
| 24 |
+
// 5173/5174. All of them must route API traffic to the backend at :8000
|
| 25 |
+
// rather than to the Vite origin (which would SPA-fallback every request
|
| 26 |
+
// to index.html and break JSON.parse).
|
| 27 |
+
const DEV_VITE_PORTS = new Set(["3000", "5173", "5174"]);
|
| 28 |
|
| 29 |
function isLocalHost(hostname: string): boolean {
|
| 30 |
return LOCAL_HOSTS.has(hostname);
|
|
|
|
| 75 |
if (stored && stored.trim()) {
|
| 76 |
const clean = stripTrailingSlash(stored.trim());
|
| 77 |
const pageIsRemote = !isLocalHost(window.location.hostname);
|
| 78 |
+
const storedUrl = (() => { try { return new URL(clean); } catch { return null; } })();
|
| 79 |
+
const storedHost = storedUrl?.hostname ?? null;
|
| 80 |
+
const storedPort = storedUrl?.port ?? "";
|
| 81 |
const storedIsLocal = storedHost !== null && isLocalHost(storedHost);
|
| 82 |
if (pageIsRemote && storedIsLocal) {
|
| 83 |
// Stale setting from a previous local dev session. Ignore it and
|
| 84 |
// let getDefaultBackendUrl() take over (same origin).
|
| 85 |
return getDefaultBackendUrl();
|
| 86 |
}
|
| 87 |
+
// Stale setting pointing at a Vite DEV port on localhost (3000 / 5173
|
| 88 |
+
// / 5174). The frontend origin serves the SPA β POSTing JSON there
|
| 89 |
+
// SPA-falls-back to index.html and the caller sees a 404, which the
|
| 90 |
+
// voice-call path reads as "feature unavailable" and flips to the
|
| 91 |
+
// chat-REST fallback. Redirect to :8000 (the default backend port)
|
| 92 |
+
// so the stored preference doesn't silently break voice-call
|
| 93 |
+
// session creation.
|
| 94 |
+
if (storedIsLocal && DEV_VITE_PORTS.has(storedPort)) {
|
| 95 |
+
return "http://localhost:8000";
|
| 96 |
+
}
|
| 97 |
return clean;
|
| 98 |
}
|
| 99 |
} catch {
|
frontend/src/ui/phone/PostCallCard.test.tsx
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Unit tests for PostCallCard.
|
| 3 |
+
*
|
| 4 |
+
* Anchored to the variant Γ missed matrix in Β§ 3 of the handoff.
|
| 5 |
+
* Each test names the invariant it protects.
|
| 6 |
+
*/
|
| 7 |
+
import React from 'react'
|
| 8 |
+
import { describe, expect, it, vi } from 'vitest'
|
| 9 |
+
import { render, screen, fireEvent } from '@testing-library/react'
|
| 10 |
+
|
| 11 |
+
import PostCallCard, { postCallCardInternals } from './PostCallCard'
|
| 12 |
+
|
| 13 |
+
const { formatDuration } = postCallCardInternals
|
| 14 |
+
|
| 15 |
+
describe('formatDuration', () => {
|
| 16 |
+
it('renders seconds under a minute', () => {
|
| 17 |
+
expect(formatDuration(5)).toBe('5s')
|
| 18 |
+
expect(formatDuration(59)).toBe('59s')
|
| 19 |
+
})
|
| 20 |
+
it('rounds to whole minutes when exact', () => {
|
| 21 |
+
expect(formatDuration(60)).toBe('1 min')
|
| 22 |
+
expect(formatDuration(720)).toBe('12 min')
|
| 23 |
+
})
|
| 24 |
+
it('renders minutes + residual seconds', () => {
|
| 25 |
+
expect(formatDuration(64)).toBe('1 min 4s')
|
| 26 |
+
})
|
| 27 |
+
})
|
| 28 |
+
|
| 29 |
+
describe('PostCallCard β expand variant (default)', () => {
|
| 30 |
+
it('renders duration + persona name', () => {
|
| 31 |
+
render(<PostCallCard durationSec={42} personaName="Vesper" />)
|
| 32 |
+
expect(screen.getByText(/Call with Vesper/i)).toBeInTheDocument()
|
| 33 |
+
expect(screen.getByText(/42s/)).toBeInTheDocument()
|
| 34 |
+
})
|
| 35 |
+
|
| 36 |
+
it('hides the transcript button when no transcript is supplied', () => {
|
| 37 |
+
render(<PostCallCard durationSec={42} personaName="Vesper" />)
|
| 38 |
+
expect(screen.queryByRole('button', { name: /view transcript/i })).not.toBeInTheDocument()
|
| 39 |
+
})
|
| 40 |
+
|
| 41 |
+
it('shows the transcript button when a transcript is supplied', () => {
|
| 42 |
+
render(
|
| 43 |
+
<PostCallCard
|
| 44 |
+
durationSec={42}
|
| 45 |
+
personaName="Vesper"
|
| 46 |
+
transcript={[{ who: 'user', text: 'hello' }]}
|
| 47 |
+
/>,
|
| 48 |
+
)
|
| 49 |
+
expect(screen.getByRole('button', { name: /view transcript/i })).toBeInTheDocument()
|
| 50 |
+
})
|
| 51 |
+
|
| 52 |
+
it('expands the transcript inline (NOT a modal) when clicked', () => {
|
| 53 |
+
render(
|
| 54 |
+
<PostCallCard
|
| 55 |
+
durationSec={42}
|
| 56 |
+
personaName="Vesper"
|
| 57 |
+
transcript={[
|
| 58 |
+
{ who: 'user', text: 'hello' },
|
| 59 |
+
{ who: 'assistant', text: 'hey there' },
|
| 60 |
+
]}
|
| 61 |
+
/>,
|
| 62 |
+
)
|
| 63 |
+
fireEvent.click(screen.getByRole('button', { name: /view transcript/i }))
|
| 64 |
+
expect(screen.getByText('hey there')).toBeInTheDocument()
|
| 65 |
+
// The only role=dialog in the tree would be a modal; assert none.
|
| 66 |
+
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
|
| 67 |
+
})
|
| 68 |
+
|
| 69 |
+
it('collapses the transcript on a second click', () => {
|
| 70 |
+
render(
|
| 71 |
+
<PostCallCard
|
| 72 |
+
durationSec={42}
|
| 73 |
+
personaName="Vesper"
|
| 74 |
+
transcript={[{ who: 'assistant', text: 'hey there' }]}
|
| 75 |
+
/>,
|
| 76 |
+
)
|
| 77 |
+
const btn = screen.getByRole('button', { name: /view transcript/i })
|
| 78 |
+
fireEvent.click(btn)
|
| 79 |
+
expect(screen.getByText('hey there')).toBeInTheDocument()
|
| 80 |
+
fireEvent.click(screen.getByRole('button', { name: /collapse transcript/i }))
|
| 81 |
+
expect(screen.queryByText('hey there')).not.toBeInTheDocument()
|
| 82 |
+
})
|
| 83 |
+
|
| 84 |
+
it('fires onResume when Resume call is clicked', () => {
|
| 85 |
+
const onResume = vi.fn()
|
| 86 |
+
render(
|
| 87 |
+
<PostCallCard durationSec={42} personaName="Vesper" onResume={onResume} />,
|
| 88 |
+
)
|
| 89 |
+
fireEvent.click(screen.getByRole('button', { name: /resume call/i }))
|
| 90 |
+
expect(onResume).toHaveBeenCalledTimes(1)
|
| 91 |
+
})
|
| 92 |
+
})
|
| 93 |
+
|
| 94 |
+
describe('PostCallCard β highlights variant', () => {
|
| 95 |
+
it('shows the summary line and a "Full transcript" button when handler supplied', () => {
|
| 96 |
+
const onOpen = vi.fn()
|
| 97 |
+
render(
|
| 98 |
+
<PostCallCard
|
| 99 |
+
durationSec={600}
|
| 100 |
+
personaName="Vesper"
|
| 101 |
+
variant="highlights"
|
| 102 |
+
summary="You talked about Lisbon and made plans for Friday."
|
| 103 |
+
onOpenFullTranscript={onOpen}
|
| 104 |
+
/>,
|
| 105 |
+
)
|
| 106 |
+
expect(screen.getByText(/Vesper remembers/i)).toBeInTheDocument()
|
| 107 |
+
expect(
|
| 108 |
+
screen.getByText(/You talked about Lisbon and made plans for Friday/i),
|
| 109 |
+
).toBeInTheDocument()
|
| 110 |
+
fireEvent.click(screen.getByRole('button', { name: /open full transcript/i }))
|
| 111 |
+
expect(onOpen).toHaveBeenCalledTimes(1)
|
| 112 |
+
})
|
| 113 |
+
|
| 114 |
+
it('omits the "Full transcript" button when no handler is supplied', () => {
|
| 115 |
+
render(
|
| 116 |
+
<PostCallCard
|
| 117 |
+
durationSec={600}
|
| 118 |
+
personaName="Vesper"
|
| 119 |
+
variant="highlights"
|
| 120 |
+
summary="β¦"
|
| 121 |
+
/>,
|
| 122 |
+
)
|
| 123 |
+
expect(
|
| 124 |
+
screen.queryByRole('button', { name: /open full transcript/i }),
|
| 125 |
+
).not.toBeInTheDocument()
|
| 126 |
+
})
|
| 127 |
+
})
|
| 128 |
+
|
| 129 |
+
describe('PostCallCard β missed variant', () => {
|
| 130 |
+
it('swaps Resume β Call back and omits secondary CTA', () => {
|
| 131 |
+
const onCallBack = vi.fn()
|
| 132 |
+
render(
|
| 133 |
+
<PostCallCard
|
| 134 |
+
durationSec={0}
|
| 135 |
+
personaName="Vesper"
|
| 136 |
+
missed
|
| 137 |
+
onCallBack={onCallBack}
|
| 138 |
+
/>,
|
| 139 |
+
)
|
| 140 |
+
expect(screen.getByText(/Missed call/i)).toBeInTheDocument()
|
| 141 |
+
fireEvent.click(screen.getByRole('button', { name: /call back/i }))
|
| 142 |
+
expect(onCallBack).toHaveBeenCalledTimes(1)
|
| 143 |
+
// No secondary button at all in the missed variant.
|
| 144 |
+
expect(screen.queryByRole('button', { name: /view transcript/i })).not.toBeInTheDocument()
|
| 145 |
+
expect(screen.queryByRole('button', { name: /full transcript/i })).not.toBeInTheDocument()
|
| 146 |
+
})
|
| 147 |
+
|
| 148 |
+
it('falls back to onResume when onCallBack is omitted', () => {
|
| 149 |
+
const onResume = vi.fn()
|
| 150 |
+
render(
|
| 151 |
+
<PostCallCard durationSec={0} personaName="Vesper" missed onResume={onResume} />,
|
| 152 |
+
)
|
| 153 |
+
fireEvent.click(screen.getByRole('button', { name: /call back/i }))
|
| 154 |
+
expect(onResume).toHaveBeenCalledTimes(1)
|
| 155 |
+
})
|
| 156 |
+
})
|
frontend/src/ui/phone/PostCallCard.tsx
ADDED
|
@@ -0,0 +1,384 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* PostCallCard β inline "we just talked" record rendered directly
|
| 3 |
+
* in the chat thread, NOT as a modal. Product direction from the
|
| 4 |
+
* engineering handoff:
|
| 5 |
+
*
|
| 6 |
+
* "You are building a conversational / companion experience,
|
| 7 |
+
* not a compliance tool. β Use inline card + expandable
|
| 8 |
+
* transcript."
|
| 9 |
+
*
|
| 10 |
+
* Three variants covering the full (variant Γ missed) matrix from
|
| 11 |
+
* the handoff's Β§ 3:
|
| 12 |
+
*
|
| 13 |
+
* expand β default. Duration + time + [Resume call] +
|
| 14 |
+
* [View transcript βΎ]. Tapping the latter expands
|
| 15 |
+
* the transcript INLINE inside the same card (no
|
| 16 |
+
* modal). Keeps the call record as an event in
|
| 17 |
+
* the conversation the user can glance at later.
|
| 18 |
+
* highlights β same shell, but the body is a short italic
|
| 19 |
+
* "Vesper remembersβ¦" summary + a [Full transcript]
|
| 20 |
+
* link. For sessions where the call was long and
|
| 21 |
+
* the full transcript is too long to inline.
|
| 22 |
+
* missed β red tint, phone-slash icon, single [Call back]
|
| 23 |
+
* CTA. Set via ``missed`` prop (orthogonal to
|
| 24 |
+
* variant; a missed call is always rendered as the
|
| 25 |
+
* missed variant, others ignored).
|
| 26 |
+
*
|
| 27 |
+
* This component replaces the prior enterprise-neutral
|
| 28 |
+
* CallMemoryCard. The enterprise form (flat surface, \"Voice session
|
| 29 |
+
* completed\" copy) is preserved as a setting-free default by
|
| 30 |
+
* passing only durationSec + endedAt; richer props turn on the
|
| 31 |
+
* conversational surface.
|
| 32 |
+
*/
|
| 33 |
+
|
| 34 |
+
import React, { useState } from 'react'
|
| 35 |
+
import {
|
| 36 |
+
IconPhone,
|
| 37 |
+
IconPhoneMissed,
|
| 38 |
+
IconSparkle,
|
| 39 |
+
} from './icons'
|
| 40 |
+
import { CALL, POST_CALL } from './tokens'
|
| 41 |
+
|
| 42 |
+
// ββ Props ββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 43 |
+
|
| 44 |
+
export type PostCallVariant = 'expand' | 'highlights'
|
| 45 |
+
|
| 46 |
+
export interface TranscriptLine {
|
| 47 |
+
who: 'user' | 'assistant'
|
| 48 |
+
text: string
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
export interface PostCallCardProps {
|
| 52 |
+
/** Live-phase duration in seconds. Rendered as "12s" / "1m 4s" /
|
| 53 |
+
* "12 min" depending on magnitude. */
|
| 54 |
+
durationSec: number
|
| 55 |
+
/** Epoch-ms timestamp of when the call ended. Rendered as
|
| 56 |
+
* "8:14 PM" via toLocaleTimeString. */
|
| 57 |
+
endedAt?: number
|
| 58 |
+
/** The persona the user was talking to. Drives the body text
|
| 59 |
+
* ("Call with Vesper"). Default "Assistant". */
|
| 60 |
+
personaName?: string
|
| 61 |
+
/** Variant β see module docstring. */
|
| 62 |
+
variant?: PostCallVariant
|
| 63 |
+
/** Missed-call variant β tints red, shows phone-slash icon,
|
| 64 |
+
* swaps the CTAs. */
|
| 65 |
+
missed?: boolean
|
| 66 |
+
/** Optional transcript. When present AND variant='expand',
|
| 67 |
+
* enables the inline expand button. */
|
| 68 |
+
transcript?: TranscriptLine[]
|
| 69 |
+
/** Optional short summary for variant='highlights'
|
| 70 |
+
* (e.g. "You talked about Lisbon and made plans for Friday"). */
|
| 71 |
+
summary?: string
|
| 72 |
+
/** "Resume call" click β re-open the call overlay with
|
| 73 |
+
* skipDialing=true. */
|
| 74 |
+
onResume?: () => void
|
| 75 |
+
/** "Call back" click on the missed variant. Defaults to the same
|
| 76 |
+
* as onResume when omitted. */
|
| 77 |
+
onCallBack?: () => void
|
| 78 |
+
/** "Full transcript" click on the highlights variant β typically
|
| 79 |
+
* opens a separate route or sheet. Optional; button hides when
|
| 80 |
+
* handler isn't provided. */
|
| 81 |
+
onOpenFullTranscript?: () => void
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
// ββ Helpers ββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 85 |
+
|
| 86 |
+
function formatDuration(s: number): string {
|
| 87 |
+
if (s < 60) return `${s}s`
|
| 88 |
+
const m = Math.floor(s / 60)
|
| 89 |
+
const r = s % 60
|
| 90 |
+
if (r === 0) return m < 60 ? `${m} min` : `${Math.floor(m / 60)}h ${m % 60}m`
|
| 91 |
+
return `${m} min ${r}s`
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
function formatClock(endedAt: number): string {
|
| 95 |
+
return new Date(endedAt).toLocaleTimeString(undefined, {
|
| 96 |
+
hour: 'numeric',
|
| 97 |
+
minute: '2-digit',
|
| 98 |
+
})
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
// ββ Component ββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 102 |
+
|
| 103 |
+
const PostCallCard: React.FC<PostCallCardProps> = ({
|
| 104 |
+
durationSec,
|
| 105 |
+
endedAt,
|
| 106 |
+
personaName = 'Assistant',
|
| 107 |
+
variant = 'expand',
|
| 108 |
+
missed = false,
|
| 109 |
+
transcript,
|
| 110 |
+
summary,
|
| 111 |
+
onResume,
|
| 112 |
+
onCallBack,
|
| 113 |
+
onOpenFullTranscript,
|
| 114 |
+
}) => {
|
| 115 |
+
const [expanded, setExpanded] = useState(false)
|
| 116 |
+
const clock = endedAt ? formatClock(endedAt) : ''
|
| 117 |
+
const dur = formatDuration(durationSec)
|
| 118 |
+
const accent = missed ? CALL.danger : CALL.rose
|
| 119 |
+
const hasTranscript = !!(transcript && transcript.length > 0)
|
| 120 |
+
|
| 121 |
+
// Top row β icon, title, sub.
|
| 122 |
+
const subtitle = missed
|
| 123 |
+
? clock
|
| 124 |
+
? `${clock} Β· tap to call back`
|
| 125 |
+
: 'tap to call back'
|
| 126 |
+
: clock
|
| 127 |
+
? `${dur} Β· ${clock}`
|
| 128 |
+
: dur
|
| 129 |
+
|
| 130 |
+
const titleText = missed
|
| 131 |
+
? `Missed call Β· ${personaName}`
|
| 132 |
+
: `Call with ${personaName}`
|
| 133 |
+
|
| 134 |
+
return (
|
| 135 |
+
<div
|
| 136 |
+
role="note"
|
| 137 |
+
aria-label={missed ? `Missed call from ${personaName}` : `Call with ${personaName}`}
|
| 138 |
+
className="hp-fade-in"
|
| 139 |
+
style={{
|
| 140 |
+
width: POST_CALL.cardWidth,
|
| 141 |
+
maxWidth: '100%',
|
| 142 |
+
borderRadius: POST_CALL.cardRadius,
|
| 143 |
+
overflow: 'hidden',
|
| 144 |
+
background: missed
|
| 145 |
+
? `linear-gradient(180deg, color-mix(in oklch, ${accent} 12%, transparent) 0%, rgba(245,236,255,0.03) 100%)`
|
| 146 |
+
: 'linear-gradient(180deg, rgba(245,236,255,0.06) 0%, rgba(245,236,255,0.03) 100%)',
|
| 147 |
+
border: missed
|
| 148 |
+
? `0.5px solid color-mix(in oklch, ${accent} 35%, transparent)`
|
| 149 |
+
: `0.5px solid ${CALL.line}`,
|
| 150 |
+
fontFamily: CALL.font,
|
| 151 |
+
color: CALL.ink,
|
| 152 |
+
}}
|
| 153 |
+
>
|
| 154 |
+
{/* Header row β icon + title + time */}
|
| 155 |
+
<div
|
| 156 |
+
style={{
|
| 157 |
+
padding: POST_CALL.rowPadding,
|
| 158 |
+
display: 'flex',
|
| 159 |
+
alignItems: 'center',
|
| 160 |
+
gap: 10,
|
| 161 |
+
}}
|
| 162 |
+
>
|
| 163 |
+
<div
|
| 164 |
+
aria-hidden="true"
|
| 165 |
+
style={{
|
| 166 |
+
width: POST_CALL.avatarSize,
|
| 167 |
+
height: POST_CALL.avatarSize,
|
| 168 |
+
borderRadius: '50%',
|
| 169 |
+
background: `color-mix(in oklch, ${accent} 20%, transparent)`,
|
| 170 |
+
border: `0.5px solid color-mix(in oklch, ${accent} 40%, transparent)`,
|
| 171 |
+
display: 'flex',
|
| 172 |
+
alignItems: 'center',
|
| 173 |
+
justifyContent: 'center',
|
| 174 |
+
color: accent,
|
| 175 |
+
flexShrink: 0,
|
| 176 |
+
}}
|
| 177 |
+
>
|
| 178 |
+
{missed ? (
|
| 179 |
+
<IconPhoneMissed size={14} color={accent} />
|
| 180 |
+
) : (
|
| 181 |
+
<IconPhone size={14} color={accent} />
|
| 182 |
+
)}
|
| 183 |
+
</div>
|
| 184 |
+
<div style={{ flex: 1, minWidth: 0 }}>
|
| 185 |
+
<div
|
| 186 |
+
style={{
|
| 187 |
+
fontSize: 13,
|
| 188 |
+
fontWeight: 600,
|
| 189 |
+
letterSpacing: -0.1,
|
| 190 |
+
color: missed ? accent : CALL.ink,
|
| 191 |
+
whiteSpace: 'nowrap',
|
| 192 |
+
overflow: 'hidden',
|
| 193 |
+
textOverflow: 'ellipsis',
|
| 194 |
+
}}
|
| 195 |
+
>
|
| 196 |
+
{titleText}
|
| 197 |
+
</div>
|
| 198 |
+
<div
|
| 199 |
+
style={{
|
| 200 |
+
fontSize: 11,
|
| 201 |
+
color: CALL.dim,
|
| 202 |
+
fontVariantNumeric: 'tabular-nums',
|
| 203 |
+
}}
|
| 204 |
+
>
|
| 205 |
+
{subtitle}
|
| 206 |
+
</div>
|
| 207 |
+
</div>
|
| 208 |
+
</div>
|
| 209 |
+
|
| 210 |
+
{/* Highlights body β italic summary line (only when variant is
|
| 211 |
+
'highlights' and we're not in the missed state). */}
|
| 212 |
+
{variant === 'highlights' && !missed && summary ? (
|
| 213 |
+
<div
|
| 214 |
+
style={{
|
| 215 |
+
padding: '4px 14px 12px',
|
| 216 |
+
fontFamily: CALL.display,
|
| 217 |
+
fontSize: 15,
|
| 218 |
+
lineHeight: 1.4,
|
| 219 |
+
color: 'rgba(245,236,255,0.82)',
|
| 220 |
+
fontStyle: 'italic',
|
| 221 |
+
}}
|
| 222 |
+
>
|
| 223 |
+
<div
|
| 224 |
+
style={{
|
| 225 |
+
display: 'flex',
|
| 226 |
+
alignItems: 'center',
|
| 227 |
+
gap: 5,
|
| 228 |
+
marginBottom: 4,
|
| 229 |
+
fontFamily: CALL.font,
|
| 230 |
+
fontStyle: 'normal',
|
| 231 |
+
fontSize: 9,
|
| 232 |
+
color: 'oklch(0.80 0.15 340)',
|
| 233 |
+
letterSpacing: 1.4,
|
| 234 |
+
textTransform: 'uppercase',
|
| 235 |
+
}}
|
| 236 |
+
>
|
| 237 |
+
<IconSparkle size={9} color="oklch(0.80 0.15 340)" />
|
| 238 |
+
{personaName} remembers
|
| 239 |
+
</div>
|
| 240 |
+
<span>“{summary}”</span>
|
| 241 |
+
</div>
|
| 242 |
+
) : null}
|
| 243 |
+
|
| 244 |
+
{/* Expanded transcript β inline, NOT a modal. Renders only for
|
| 245 |
+
variant='expand' when the user has clicked the expand button
|
| 246 |
+
and a transcript was provided. */}
|
| 247 |
+
{variant === 'expand' && expanded && !missed && hasTranscript ? (
|
| 248 |
+
<div
|
| 249 |
+
style={{
|
| 250 |
+
padding: '2px 14px 12px',
|
| 251 |
+
borderTop: `0.5px solid ${CALL.line}`,
|
| 252 |
+
display: 'flex',
|
| 253 |
+
flexDirection: 'column',
|
| 254 |
+
gap: 8,
|
| 255 |
+
}}
|
| 256 |
+
>
|
| 257 |
+
<div
|
| 258 |
+
style={{
|
| 259 |
+
fontSize: 9,
|
| 260 |
+
letterSpacing: 1.4,
|
| 261 |
+
textTransform: 'uppercase',
|
| 262 |
+
color: 'rgba(245,236,255,0.4)',
|
| 263 |
+
paddingTop: 10,
|
| 264 |
+
}}
|
| 265 |
+
>
|
| 266 |
+
Transcript
|
| 267 |
+
</div>
|
| 268 |
+
{(transcript as TranscriptLine[]).map((line, i) => (
|
| 269 |
+
<div key={i} style={{ fontSize: 12, lineHeight: 1.45 }}>
|
| 270 |
+
<span
|
| 271 |
+
style={{
|
| 272 |
+
color:
|
| 273 |
+
line.who === 'user'
|
| 274 |
+
? 'oklch(0.80 0.15 50)'
|
| 275 |
+
: 'oklch(0.80 0.15 340)',
|
| 276 |
+
fontWeight: 600,
|
| 277 |
+
marginRight: 6,
|
| 278 |
+
}}
|
| 279 |
+
>
|
| 280 |
+
{line.who === 'user' ? 'You' : personaName}
|
| 281 |
+
</span>
|
| 282 |
+
<span style={{ color: 'rgba(245,236,255,0.85)' }}>
|
| 283 |
+
{line.text}
|
| 284 |
+
</span>
|
| 285 |
+
</div>
|
| 286 |
+
))}
|
| 287 |
+
</div>
|
| 288 |
+
) : null}
|
| 289 |
+
|
| 290 |
+
{/* CTA row */}
|
| 291 |
+
<div
|
| 292 |
+
style={{
|
| 293 |
+
display: 'flex',
|
| 294 |
+
gap: 6,
|
| 295 |
+
padding: '8px 10px 10px',
|
| 296 |
+
borderTop:
|
| 297 |
+
variant === 'expand' && expanded && !missed && hasTranscript
|
| 298 |
+
? 'none'
|
| 299 |
+
: `0.5px solid ${CALL.line}`,
|
| 300 |
+
}}
|
| 301 |
+
>
|
| 302 |
+
<button
|
| 303 |
+
type="button"
|
| 304 |
+
onClick={missed ? (onCallBack ?? onResume) : onResume}
|
| 305 |
+
disabled={!(missed ? (onCallBack ?? onResume) : onResume)}
|
| 306 |
+
aria-label={missed ? 'Call back' : 'Resume call'}
|
| 307 |
+
style={{
|
| 308 |
+
flex: 1,
|
| 309 |
+
height: POST_CALL.ctaHeight,
|
| 310 |
+
borderRadius: POST_CALL.ctaRadius,
|
| 311 |
+
cursor: 'pointer',
|
| 312 |
+
background: accent,
|
| 313 |
+
color: '#0c0810',
|
| 314 |
+
border: 'none',
|
| 315 |
+
fontFamily: CALL.font,
|
| 316 |
+
fontSize: 12,
|
| 317 |
+
fontWeight: 600,
|
| 318 |
+
display: 'flex',
|
| 319 |
+
alignItems: 'center',
|
| 320 |
+
justifyContent: 'center',
|
| 321 |
+
gap: 6,
|
| 322 |
+
}}
|
| 323 |
+
>
|
| 324 |
+
<IconPhone size={12} color="#0c0810" />
|
| 325 |
+
{missed ? 'Call back' : 'Resume call'}
|
| 326 |
+
</button>
|
| 327 |
+
|
| 328 |
+
{/* Secondary CTA β only the non-missed variants have one */}
|
| 329 |
+
{!missed && variant === 'highlights' && onOpenFullTranscript ? (
|
| 330 |
+
<button
|
| 331 |
+
type="button"
|
| 332 |
+
onClick={onOpenFullTranscript}
|
| 333 |
+
aria-label="Open full transcript"
|
| 334 |
+
style={{
|
| 335 |
+
flex: 1,
|
| 336 |
+
height: POST_CALL.ctaHeight,
|
| 337 |
+
borderRadius: POST_CALL.ctaRadius,
|
| 338 |
+
cursor: 'pointer',
|
| 339 |
+
background: 'rgba(245,236,255,0.06)',
|
| 340 |
+
color: CALL.ink,
|
| 341 |
+
border: `0.5px solid ${CALL.line}`,
|
| 342 |
+
fontFamily: CALL.font,
|
| 343 |
+
fontSize: 12,
|
| 344 |
+
fontWeight: 500,
|
| 345 |
+
}}
|
| 346 |
+
>
|
| 347 |
+
Full transcript
|
| 348 |
+
</button>
|
| 349 |
+
) : null}
|
| 350 |
+
|
| 351 |
+
{!missed && variant === 'expand' && hasTranscript ? (
|
| 352 |
+
<button
|
| 353 |
+
type="button"
|
| 354 |
+
onClick={() => setExpanded((v) => !v)}
|
| 355 |
+
aria-expanded={expanded}
|
| 356 |
+
aria-label={expanded ? 'Collapse transcript' : 'View transcript'}
|
| 357 |
+
style={{
|
| 358 |
+
flex: 1,
|
| 359 |
+
height: POST_CALL.ctaHeight,
|
| 360 |
+
borderRadius: POST_CALL.ctaRadius,
|
| 361 |
+
cursor: 'pointer',
|
| 362 |
+
background: 'rgba(245,236,255,0.06)',
|
| 363 |
+
color: CALL.ink,
|
| 364 |
+
border: `0.5px solid ${CALL.line}`,
|
| 365 |
+
fontFamily: CALL.font,
|
| 366 |
+
fontSize: 12,
|
| 367 |
+
fontWeight: 500,
|
| 368 |
+
}}
|
| 369 |
+
>
|
| 370 |
+
{expanded ? 'Collapse β΄' : 'View transcript βΎ'}
|
| 371 |
+
</button>
|
| 372 |
+
) : null}
|
| 373 |
+
</div>
|
| 374 |
+
</div>
|
| 375 |
+
)
|
| 376 |
+
}
|
| 377 |
+
|
| 378 |
+
export default PostCallCard
|
| 379 |
+
|
| 380 |
+
// Test hooks β exposed for unit tests without widening the public API.
|
| 381 |
+
export const postCallCardInternals = {
|
| 382 |
+
formatDuration,
|
| 383 |
+
formatClock,
|
| 384 |
+
}
|
frontend/src/ui/phone/README.md
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# `ui/phone/` β call surface component library
|
| 2 |
+
|
| 3 |
+
Status: **partial port** of the design system from the engineering
|
| 4 |
+
handoff (`Phone Call UI.html`). This README tracks what ships today
|
| 5 |
+
and what's queued for follow-up sessions.
|
| 6 |
+
|
| 7 |
+
## What's here now
|
| 8 |
+
|
| 9 |
+
```
|
| 10 |
+
phone/
|
| 11 |
+
βββ README.md this file
|
| 12 |
+
βββ tokens.ts CALL palette + POST_CALL layout constants
|
| 13 |
+
βββ icons.tsx shared SVG icon set (mic, phone*, chat, β¦)
|
| 14 |
+
βββ PostCallCard.tsx inline chat card, expandable transcript,
|
| 15 |
+
β 3 variants (expand / highlights / missed)
|
| 16 |
+
βββ PostCallCard.test.tsx 13 tests β variant matrix + keyboard/click
|
| 17 |
+
βββ primitives/
|
| 18 |
+
βββ index.ts barrel β single import path
|
| 19 |
+
βββ useReducedMotion.ts OS motion-preference subscription
|
| 20 |
+
βββ useReducedMotion.test.ts 4 tests
|
| 21 |
+
βββ useFocusTrap.ts keyboard-focus containment for dialogs
|
| 22 |
+
βββ useFocusTrap.test.tsx 6 tests
|
| 23 |
+
βββ ControlBtn.tsx circular action button
|
| 24 |
+
βββ ControlBtn.test.tsx 8 tests
|
| 25 |
+
βββ Waveform.tsx rAF-driven audio-level bars
|
| 26 |
+
βββ Waveform.test.tsx 9 tests
|
| 27 |
+
βββ Aura.tsx seeded-hue persona identity chip
|
| 28 |
+
βββ Aura.test.tsx 8 tests
|
| 29 |
+
βββ AmbientAura.tsx page-scale coloured backdrop glow
|
| 30 |
+
βββ AmbientAura.test.tsx 7 tests
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
Consumed by:
|
| 34 |
+
|
| 35 |
+
- `frontend/src/ui/App.tsx` β renders `PostCallCard` inline in the
|
| 36 |
+
chat stream for every `Msg` that carries a `callMemory` payload.
|
| 37 |
+
The card is **not** a modal β it sits in the conversation so the
|
| 38 |
+
user can expand the transcript without a context switch.
|
| 39 |
+
- `primitives/` are additive foundation for the next batch
|
| 40 |
+
(Waveform, Aura, AmbientAura, CallScreen). No consumers yet;
|
| 41 |
+
`CallOverlay` still has its own inlined versions of waveform +
|
| 42 |
+
avatar + control buttons, which the next session extracts.
|
| 43 |
+
|
| 44 |
+
## What's queued for follow-up sessions
|
| 45 |
+
|
| 46 |
+
The handoff defines a much larger set of components. They are
|
| 47 |
+
additive, so shipping them one at a time doesn't block the current
|
| 48 |
+
call feature β it's already functional end-to-end with the
|
| 49 |
+
`CallOverlay` + the inline `PostCallCard`.
|
| 50 |
+
|
| 51 |
+
| File | Status | Notes |
|
| 52 |
+
|---|---|---|
|
| 53 |
+
| `primitives/ControlBtn.tsx` | **Shipped** | 8 tests. CallOverlay still inlines its own; extraction queued behind CallScreen. |
|
| 54 |
+
| `primitives/useReducedMotion.ts` | **Shipped** | 4 tests. Drives the motion gate on Waveform + Aura + AmbientAura. |
|
| 55 |
+
| `primitives/useFocusTrap.ts` | **Shipped** | 6 tests. Consumed by CallScreen + LockScreenIncoming when they land. |
|
| 56 |
+
| `primitives/Waveform.tsx` | **Shipped + extracted** | 9 tests. CallOverlay's inline CallWaveform is gone β `<Waveform mode={waveformModeFromCallState(state)} intensityRef={β¦} />` takes its place. |
|
| 57 |
+
| `primitives/Aura.tsx` | **Shipped + extracted** | 8 tests. FNV-1a hashed seeded hue + optional photoUrl path. Inner disc of CallOverlay's CallAvatar now delegates to `<Aura />`; the halo + pulse rings stay outside (state-dependent). |
|
| 58 |
+
| `primitives/AmbientAura.tsx` | **Shipped** | 7 tests. Additive β CallOverlay today uses a flat dim+blur backdrop; this primitive is slated for CallScreen's fullscreen composition. |
|
| 59 |
+
| `CallScreen.tsx` | Pending | Fullscreen mobile composition. Desktop uses the existing `CallOverlay` modal. |
|
| 60 |
+
| `CallOverlay.tsx` (refactor) | Pending | Route between fullscreen / modal / PiP based on viewport. Today's `CallOverlay.tsx` is desktop-modal-only. |
|
| 61 |
+
| `TextInCall.tsx` | Pending | Mid-call composer sheet β minimize call to a thin bar, slide composer up, send text over the existing chat REST. |
|
| 62 |
+
| `HangUpTransition.tsx` | Pending | ~1 s "call ended" fade beat before the overlay dismisses. Today the overlay fades inline. |
|
| 63 |
+
| `MutedToast.tsx` | Pending | Brief "you're muted" confirmation pill. Today the muted state is indicated only by the waveform colour. |
|
| 64 |
+
| `LockScreenIncoming.tsx` | Pending | iOS-style full-bleed incoming with slide-to-answer. PWA only. |
|
| 65 |
+
| `PoorConnection.tsx` | Pending | Reconnecting banner when WS has been silent >3 s. Today the socket reconnects silently. |
|
| 66 |
+
|
| 67 |
+
## Suggested sequencing for the next session
|
| 68 |
+
|
| 69 |
+
Small commits, one per component file, matching the pattern from
|
| 70 |
+
this session:
|
| 71 |
+
|
| 72 |
+
1. `primitives/Waveform.tsx` β extract the `rAF`-driven version
|
| 73 |
+
from `CallOverlay.tsx`, swap `CallOverlay`'s inline copy for
|
| 74 |
+
the import.
|
| 75 |
+
2. `primitives/Aura.tsx` β extract `CallAvatar`, thread through
|
| 76 |
+
`CallOverlay` + `PostCallCard` so the avatar identity is
|
| 77 |
+
consistent across surfaces.
|
| 78 |
+
3. `primitives/ControlBtn.tsx` β same.
|
| 79 |
+
4. `CallScreen.tsx` β fullscreen mobile composition on top of the
|
| 80 |
+
primitives.
|
| 81 |
+
5. `HangUpTransition.tsx` + `MutedToast.tsx` β small polish
|
| 82 |
+
additions; each a single new render branch in `CallOverlay`.
|
| 83 |
+
6. `TextInCall.tsx` β biggest piece, needs its own focus session.
|
| 84 |
+
7. `LockScreenIncoming.tsx` β PWA-only, coordinates with service-
|
| 85 |
+
worker push; scope-separate.
|
| 86 |
+
8. `PoorConnection.tsx` β layer on `callSocket`'s existing silence
|
| 87 |
+
watchdog.
|
| 88 |
+
|
| 89 |
+
Keep each batch under ~300 LOC and write a vitest for every
|
| 90 |
+
component (the 13-test PostCallCard file is the template).
|
| 91 |
+
|
| 92 |
+
## Design reference
|
| 93 |
+
|
| 94 |
+
The pixel-level source of truth is `Phone Call UI.html` in the
|
| 95 |
+
design project β matches every state and surface. Each TSX file in
|
| 96 |
+
this folder maps 1:1 to a named component in the design canvas.
|
| 97 |
+
|
| 98 |
+
## Not in scope (anywhere in this folder)
|
| 99 |
+
|
| 100 |
+
- New backend endpoints
|
| 101 |
+
- Replacing the `voice_call` module
|
| 102 |
+
- Replacing `CallOverlay`'s streaming / barge-in plumbing
|
| 103 |
+
- Group calls / add participant
|
frontend/src/ui/phone/icons.tsx
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* phone/icons.tsx β shared SVG icon set for the call surface.
|
| 3 |
+
*
|
| 4 |
+
* Direct-return functions (no shared base helper) so each export is
|
| 5 |
+
* trivially tree-shakable and the module's export table stays
|
| 6 |
+
* predictable under the vitest transform (a helper-based variant
|
| 7 |
+
* earlier in this branch caused every export to vanish from the
|
| 8 |
+
* module namespace β the root cause wasn't isolated, but the
|
| 9 |
+
* direct-return form sidesteps it entirely).
|
| 10 |
+
*
|
| 11 |
+
* Stroke weight is 1.6 for mic/chat/speaker/video (secondary
|
| 12 |
+
* actions) and 1.8 for phone icons + back (primary + nav) to match
|
| 13 |
+
* the design spec in ``Phone Call UI.html``. Icons inherit
|
| 14 |
+
* currentColor by default so a parent text colour change cascades
|
| 15 |
+
* cleanly.
|
| 16 |
+
*/
|
| 17 |
+
|
| 18 |
+
import React from 'react'
|
| 19 |
+
|
| 20 |
+
interface IconProps {
|
| 21 |
+
size?: number
|
| 22 |
+
color?: string
|
| 23 |
+
title?: string
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
type SvgExtras = {
|
| 27 |
+
role: 'img' | 'presentation'
|
| 28 |
+
'aria-hidden'?: 'true'
|
| 29 |
+
'aria-label'?: string
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
function a11y(title?: string): SvgExtras {
|
| 33 |
+
return title
|
| 34 |
+
? { role: 'img', 'aria-label': title }
|
| 35 |
+
: { role: 'presentation', 'aria-hidden': 'true' }
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
// ββ Icons βββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 39 |
+
|
| 40 |
+
export const IconMic: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 41 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 42 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 43 |
+
{...a11y(title)}>
|
| 44 |
+
{title ? <title>{title}</title> : null}
|
| 45 |
+
<rect x="9" y="3" width="6" height="12" rx="3" />
|
| 46 |
+
<path d="M5 11a7 7 0 0 0 14 0M12 18v3" />
|
| 47 |
+
</svg>
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
export const IconMicOff: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 51 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 52 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 53 |
+
{...a11y(title)}>
|
| 54 |
+
{title ? <title>{title}</title> : null}
|
| 55 |
+
<path d="M9 9V6a3 3 0 0 1 6 0v5m0 4a3 3 0 0 1-6 0" />
|
| 56 |
+
<path d="M5 11a7 7 0 0 0 11.5 5.3M19 11a7 7 0 0 1-.4 2.3M12 18v3" />
|
| 57 |
+
<path d="M3 3l18 18" />
|
| 58 |
+
</svg>
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
export const IconPhone: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 62 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 63 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 64 |
+
{...a11y(title)}>
|
| 65 |
+
{title ? <title>{title}</title> : null}
|
| 66 |
+
<path d="M5 4h3l2 5-2.5 1.5a11 11 0 0 0 6 6L15 14l5 2v3a2 2 0 0 1-2 2A16 16 0 0 1 3 6a2 2 0 0 1 2-2z" />
|
| 67 |
+
</svg>
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
export const IconPhoneEnd: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 71 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 72 |
+
stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
|
| 73 |
+
{...a11y(title)}>
|
| 74 |
+
{title ? <title>{title}</title> : null}
|
| 75 |
+
<path d="M4 14c5-5 11-5 16 0l-2 2-3-1v-2a9 9 0 0 0-6 0v2l-3 1-2-2z"
|
| 76 |
+
transform="rotate(135 12 12)" />
|
| 77 |
+
</svg>
|
| 78 |
+
)
|
| 79 |
+
|
| 80 |
+
/** Phone with superimposed slash β used by PostCallCard's missed
|
| 81 |
+
* variant so the missed state reads differently from a normal
|
| 82 |
+
* call event even at thumbnail sizes. */
|
| 83 |
+
export const IconPhoneMissed: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 84 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 85 |
+
stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
|
| 86 |
+
{...a11y(title)}>
|
| 87 |
+
{title ? <title>{title}</title> : null}
|
| 88 |
+
<path d="M22 16.92v3a2 2 0 01-2.18 2 19.79 19.79 0 01-8.63-3.07 19.5 19.5 0 01-6-6 19.79 19.79 0 01-3.07-8.67A2 2 0 014.11 2h3a2 2 0 012 1.72 12.84 12.84 0 00.7 2.81 2 2 0 01-.45 2.11L8.09 9.91a16 16 0 006 6l1.27-1.27a2 2 0 012.11-.45 12.84 12.84 0 002.81.7 2 2 0 011.72 2.03z" />
|
| 89 |
+
<line x1="23" y1="1" x2="17" y2="7" />
|
| 90 |
+
<line x1="17" y1="1" x2="23" y2="7" />
|
| 91 |
+
</svg>
|
| 92 |
+
)
|
| 93 |
+
|
| 94 |
+
export const IconChat: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 95 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 96 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 97 |
+
{...a11y(title)}>
|
| 98 |
+
{title ? <title>{title}</title> : null}
|
| 99 |
+
<path d="M4 5h16v11H9l-5 4z" />
|
| 100 |
+
<circle cx="9" cy="10.5" r="0.9" fill={color} stroke="none" />
|
| 101 |
+
<circle cx="12" cy="10.5" r="0.9" fill={color} stroke="none" />
|
| 102 |
+
<circle cx="15" cy="10.5" r="0.9" fill={color} stroke="none" />
|
| 103 |
+
</svg>
|
| 104 |
+
)
|
| 105 |
+
|
| 106 |
+
export const IconSpeaker: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 107 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 108 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 109 |
+
{...a11y(title)}>
|
| 110 |
+
{title ? <title>{title}</title> : null}
|
| 111 |
+
<path d="M4 9v6h4l5 4V5L8 9H4z" />
|
| 112 |
+
<path d="M17 8a5 5 0 0 1 0 8M19.5 5.5a8.5 8.5 0 0 1 0 13" />
|
| 113 |
+
</svg>
|
| 114 |
+
)
|
| 115 |
+
|
| 116 |
+
export const IconVideo: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 117 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 118 |
+
stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
|
| 119 |
+
{...a11y(title)}>
|
| 120 |
+
{title ? <title>{title}</title> : null}
|
| 121 |
+
<rect x="3" y="6" width="13" height="12" rx="2" />
|
| 122 |
+
<path d="M16 10l5-3v10l-5-3z" />
|
| 123 |
+
</svg>
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
export const IconBack: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
|
| 127 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill="none"
|
| 128 |
+
stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
|
| 129 |
+
{...a11y(title)}>
|
| 130 |
+
{title ? <title>{title}</title> : null}
|
| 131 |
+
<path d="M15 5l-7 7 7 7" />
|
| 132 |
+
</svg>
|
| 133 |
+
)
|
| 134 |
+
|
| 135 |
+
/** Small star glyph used by the "persona remembers" label on the
|
| 136 |
+
* highlights variant of PostCallCard. Fill-mode render (not stroke)
|
| 137 |
+
* reads better at 9β11 px. */
|
| 138 |
+
export const IconSparkle: React.FC<IconProps> = ({ size = 12, color = 'currentColor', title }) => (
|
| 139 |
+
<svg width={size} height={size} viewBox="0 0 24 24" fill={color}
|
| 140 |
+
{...a11y(title)}>
|
| 141 |
+
{title ? <title>{title}</title> : null}
|
| 142 |
+
<path d="M12 2l1.8 6.2L20 10l-6.2 1.8L12 18l-1.8-6.2L4 10l6.2-1.8z" />
|
| 143 |
+
</svg>
|
| 144 |
+
)
|