HomePilot Deploy Bot commited on
Commit
0ef8bc0
Β·
1 Parent(s): 4b3b11c

chore(hf): sync HomePilot to HF Space

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. .gitattributes +5 -5
  2. backend/app/main.py +38 -0
  3. backend/app/persona_call/__init__.py +33 -0
  4. backend/app/persona_call/backchannel.py +122 -0
  5. backend/app/persona_call/closing.py +102 -0
  6. backend/app/persona_call/config.py +102 -0
  7. backend/app/persona_call/context.py +191 -0
  8. backend/app/persona_call/directive.py +365 -0
  9. backend/app/persona_call/facets.py +180 -0
  10. backend/app/persona_call/latency.py +107 -0
  11. backend/app/persona_call/openings.py +334 -0
  12. backend/app/persona_call/repeat.py +128 -0
  13. backend/app/persona_call/router.py +224 -0
  14. backend/app/persona_call/state.py +99 -0
  15. backend/app/persona_call/store.py +241 -0
  16. backend/app/studio/render_jobs.py +14 -0
  17. backend/app/studio/render_mp4.py +206 -4
  18. backend/app/studio/routes.py +98 -0
  19. backend/app/voice_call/__init__.py +16 -0
  20. backend/app/voice_call/barge_in.py +123 -0
  21. backend/app/voice_call/config.py +92 -0
  22. backend/app/voice_call/models.py +112 -0
  23. backend/app/voice_call/policy.py +73 -0
  24. backend/app/voice_call/router.py +223 -0
  25. backend/app/voice_call/service.py +145 -0
  26. backend/app/voice_call/store.py +293 -0
  27. backend/app/voice_call/turn.py +125 -0
  28. backend/app/voice_call/turn_stream.py +259 -0
  29. backend/app/voice_call/ws.py +605 -0
  30. frontend/.cache-bust +1 -1
  31. frontend/src/ui/App.tsx +291 -8
  32. frontend/src/ui/CallOverlay.tsx +1389 -0
  33. frontend/src/ui/CreatorStudioEditor.tsx +6 -0
  34. frontend/src/ui/SettingsPanel.tsx +48 -20
  35. frontend/src/ui/call/bargeIn.test.ts +81 -0
  36. frontend/src/ui/call/bargeIn.ts +132 -0
  37. frontend/src/ui/call/callApi.ts +190 -0
  38. frontend/src/ui/call/callSocket.ts +579 -0
  39. frontend/src/ui/call/log.ts +140 -0
  40. frontend/src/ui/call/streamTts.test.ts +65 -0
  41. frontend/src/ui/call/streamTts.ts +173 -0
  42. frontend/src/ui/call/useCallSession.ts +381 -0
  43. frontend/src/ui/components/CreatorExportWizard.tsx +423 -65
  44. frontend/src/ui/components/TtsEngineSection.tsx +279 -0
  45. frontend/src/ui/lib/backendUrl.test.ts +35 -0
  46. frontend/src/ui/lib/backendUrl.ts +19 -2
  47. frontend/src/ui/phone/PostCallCard.test.tsx +156 -0
  48. frontend/src/ui/phone/PostCallCard.tsx +384 -0
  49. frontend/src/ui/phone/README.md +103 -0
  50. frontend/src/ui/phone/icons.tsx +144 -0
.gitattributes CHANGED
@@ -1,9 +1,9 @@
1
- *.jpeg filter=lfs diff=lfs merge=lfs -text
2
- *.woff filter=lfs diff=lfs merge=lfs -text
3
- *.ico filter=lfs diff=lfs merge=lfs -text
4
  *.hpersona filter=lfs diff=lfs merge=lfs -text
5
- *.png filter=lfs diff=lfs merge=lfs -text
6
  *.jpg filter=lfs diff=lfs merge=lfs -text
7
- *.webp filter=lfs diff=lfs merge=lfs -text
8
  *.svg filter=lfs diff=lfs merge=lfs -text
9
  *.woff2 filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
 
1
  *.hpersona filter=lfs diff=lfs merge=lfs -text
 
2
  *.jpg filter=lfs diff=lfs merge=lfs -text
 
3
  *.svg filter=lfs diff=lfs merge=lfs -text
4
  *.woff2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ico filter=lfs diff=lfs merge=lfs -text
6
+ *.png filter=lfs diff=lfs merge=lfs -text
7
+ *.jpeg filter=lfs diff=lfs merge=lfs -text
8
+ *.webp filter=lfs diff=lfs merge=lfs -text
9
+ *.woff filter=lfs diff=lfs merge=lfs -text
backend/app/main.py CHANGED
@@ -159,6 +159,44 @@ app.include_router(teams_router)
159
  from .teams.bridge_routes import router as teams_bridge_router
160
  app.include_router(teams_bridge_router)
161
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
162
  # Include Marketplace routes (/v1/marketplace/*)
163
  app.include_router(marketplace_router)
164
 
 
159
  from .teams.bridge_routes import router as teams_bridge_router
160
  app.include_router(teams_bridge_router)
161
 
162
+ # --- Voice call (/v1/voice-call/*) β€” ADDITIVE, FEATURE-FLAGGED ---------------
163
+ # Fully off by default. Enable with ``VOICE_CALL_ENABLED=true``. If anything
164
+ # in the voice_call module raises on import (bad migration, missing dep, …)
165
+ # we swallow it so the rest of the app stays up β€” this feature is optional
166
+ # and must never take production down.
167
+ try:
168
+ from .voice_call.config import load as _voice_call_load_config
169
+ _voice_call_cfg = _voice_call_load_config()
170
+ if _voice_call_cfg.enabled:
171
+ from .voice_call.router import build_router as _build_vc_router
172
+ from .voice_call.router import build_ws_router as _build_vc_ws_router
173
+ app.include_router(_build_vc_router(_voice_call_cfg))
174
+ app.include_router(_build_vc_ws_router(_voice_call_cfg))
175
+ print("[voice_call] enabled β€” routes mounted under /v1/voice-call")
176
+ except Exception as _vc_err: # noqa: BLE001
177
+ # Deliberate broad catch: this entire feature is optional. Log and move on.
178
+ print(f"[voice_call] DISABLED due to import error: {_vc_err}")
179
+
180
+ # --- Persona call (/v1/persona-call/*) β€” ADDITIVE, FEATURE-FLAGGED -----------
181
+ # Phone-call conversational dynamics + per-turn system suffix composer.
182
+ # Off by default. Enable with ``PERSONA_CALL_ENABLED=true``. The module
183
+ # operates behind voice_call/ws.py β€” the HTTP router is minor (facets
184
+ # lookup, state inspection, shadow-directive audit). Independent flag so
185
+ # it can ship in shadow-compose mode (``PERSONA_CALL_APPLY=false``)
186
+ # before any prompt surface actually changes.
187
+ try:
188
+ from .persona_call.config import load as _persona_call_load_config
189
+ _persona_call_cfg = _persona_call_load_config()
190
+ if _persona_call_cfg.enabled:
191
+ from .persona_call.router import build_router as _build_pc_router
192
+ app.include_router(_build_pc_router(_persona_call_cfg))
193
+ print(
194
+ f"[persona_call] enabled (apply={_persona_call_cfg.apply}) β€” "
195
+ "routes mounted under /v1/persona-call"
196
+ )
197
+ except Exception as _pc_err: # noqa: BLE001
198
+ print(f"[persona_call] DISABLED due to import error: {_pc_err}")
199
+
200
  # Include Marketplace routes (/v1/marketplace/*)
201
  app.include_router(marketplace_router)
202
 
backend/app/persona_call/__init__.py ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ persona_call β€” phone-call conversational dynamics on top of voice_call.
3
+
4
+ Adds nothing to a persona's BASE system prompt. Instead, when a persona
5
+ is on a live phone call, this module composes a small per-turn system
6
+ *suffix* that:
7
+
8
+ * tells the persona it is currently on a phone call (not chat),
9
+ * applies one research-grounded rule from the design doc (e.g.
10
+ "skip 'how are you' after 9 pm local"),
11
+ * enforces the two-phase closing handshake at the state machine level
12
+ (the only place the model is overridden rather than advised),
13
+ * emits server-originated *backchannels* and *fillers* on a separate
14
+ low-volume stream so the user hears continuers + thinking sounds
15
+ without waiting for a full LLM turn.
16
+
17
+ Fully additive + feature-flagged. With ``PERSONA_CALL_ENABLED=false``
18
+ (the default) this module is never imported and no row is ever written
19
+ to the ``persona_call_state`` table.
20
+
21
+ Published research baked in:
22
+ Schegloff 1968, Schegloff & Sacks 1973, Sacks/Schegloff/Jefferson 1974,
23
+ Jefferson 1984, Stivers et al. 2009, Clark & French 1981,
24
+ Local & Walker 2012, OpenAI Realtime Prompting Guide, Sesame 2025.
25
+
26
+ Call-centre software-engineering practice applied:
27
+ single-source-of-truth state machine; every behavior has a named
28
+ module and a test anchor; deterministic telemetry counters for
29
+ naturalness SLOs; zero coupling to persona storage.
30
+ """
31
+ from __future__ import annotations
32
+
33
+ __all__ = ["config", "directive", "router"]
backend/app/persona_call/backchannel.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Server-originated backchannel emission β€” the separate low-volume
3
+ stream (per the user's chosen default).
4
+
5
+ A backchannel is NOT a turn. It's a "mm-hm" / "yeah" / "got it" the
6
+ server emits *while the user is still speaking* so the caller hears
7
+ recipiency instead of silence. Research (Jefferson 1984) shows humans
8
+ emit one every 2–3 clauses on the phone β€” they compensate for the
9
+ absence of visual nods.
10
+
11
+ Design rules baked in:
12
+
13
+ * Token pulled from the persona's own pool (VoiceFacets.ack_tokens)
14
+ so Darkangel666 sounds like Darkangel666 even when backchanneling.
15
+ * Cadence: at most one per ``backchannel_min_gap_ms``, and only after
16
+ ``backchannel_clause_trigger`` clause boundaries in the CURRENT
17
+ user utterance.
18
+ * Anti-repetition: never the same token two backchannels in a row.
19
+ * Played at ``backchannel_volume_db`` below the main TTS stream so
20
+ it overlaps the user's voice without fighting it (the client
21
+ applies the gain β€” this module just names the dB).
22
+ * Skipped entirely when ``facets.backchannel_rate`` is 0.
23
+
24
+ The emission itself is a WS event; this module picks the token and
25
+ decides IF/WHEN. The actual ``ws.send`` call lives in ws.py.
26
+ """
27
+ from __future__ import annotations
28
+
29
+ import random
30
+ import re
31
+ import time
32
+ from typing import Any, Dict, List, Optional, Tuple
33
+
34
+ from .config import PersonaCallConfig
35
+ from .facets import VoiceFacets
36
+
37
+
38
+ # Rough clause boundaries β€” commas, semicolons, sentence enders,
39
+ # conjunctions that start a clause ("and then…", "but…", "so…").
40
+ # Runs on the partial STT transcript, so it fires as the user speaks.
41
+ _CLAUSE_BOUNDARY = re.compile(
42
+ r"("
43
+ r"[,;]|[.!?]\s+|" # punctuation
44
+ r"\b(and|but|so|because|then|however|actually)\b\s" # discourse markers
45
+ r")",
46
+ re.I,
47
+ )
48
+
49
+
50
+ def count_clause_boundaries(partial_text: str) -> int:
51
+ """How many clause boundaries have been uttered in the current
52
+ user turn so far."""
53
+ if not partial_text:
54
+ return 0
55
+ return len(_CLAUSE_BOUNDARY.findall(partial_text))
56
+
57
+
58
+ def _now_ms() -> int:
59
+ return int(time.time() * 1000)
60
+
61
+
62
+ def choose_token(
63
+ facets: VoiceFacets,
64
+ *,
65
+ recent_acks: List[str],
66
+ seed_utterance: str = "",
67
+ ) -> Optional[str]:
68
+ """Pick a backchannel token that is (a) in the persona's pool and
69
+ (b) not the most recent one. Returns None iff the pool is empty.
70
+
71
+ Mild stance-sensitivity: if the partial utterance ended with a
72
+ rising question mark, we bias toward stance tokens ('oh really?',
73
+ 'yeah?'); falling-period endings bias toward acknowledgments.
74
+ """
75
+ pool = [t for t in facets.ack_tokens if t]
76
+ if not pool:
77
+ return None
78
+ last = (recent_acks[-1].lower() if recent_acks else "")
79
+ candidates = [t for t in pool if t.lower() != last] or pool
80
+ # Stance nudge β€” prefer single-syllable quick tokens on
81
+ # exclamations / questions, longer ones on plain pauses.
82
+ is_query = bool(seed_utterance.rstrip().endswith(("?", "!")))
83
+ if is_query:
84
+ candidates = sorted(candidates, key=lambda t: len(t))
85
+ return random.choice(candidates)
86
+
87
+
88
+ def should_emit(
89
+ *,
90
+ partial_text: str,
91
+ last_backchannel_ms: int,
92
+ facets: VoiceFacets,
93
+ cfg: PersonaCallConfig,
94
+ ) -> bool:
95
+ """True iff we should send a backchannel right now. Pure decision;
96
+ no side effects."""
97
+ if facets.backchannel_rate <= 0:
98
+ return False
99
+ # Scale the clause threshold by the persona's own rate. Low-rate
100
+ # personas (e.g. reserved / clinical) need more clauses before a
101
+ # backchannel; chatty personas need fewer.
102
+ trigger = max(
103
+ 1, int(round(cfg.backchannel_clause_trigger / max(0.25, facets.backchannel_rate)))
104
+ )
105
+ if count_clause_boundaries(partial_text) < trigger:
106
+ return False
107
+ min_gap = int(cfg.backchannel_min_gap_ms / max(0.25, facets.backchannel_rate))
108
+ if _now_ms() - int(last_backchannel_ms or 0) < min_gap:
109
+ return False
110
+ return True
111
+
112
+
113
+ def build_event(token: str, volume_db: float) -> Dict[str, Any]:
114
+ """Shape the WS event payload ws.py will serialize."""
115
+ return {
116
+ "type": "assistant.backchannel",
117
+ "payload": {
118
+ "token": token,
119
+ "volume_db": float(volume_db),
120
+ "ts": _now_ms(),
121
+ },
122
+ }
backend/app/persona_call/closing.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Two-phase closing handshake β€” the one place we *enforce* instead of
3
+ advise.
4
+
5
+ Why enforced: LLMs reliably collapse Schegloff's 3–6-turn closing
6
+ into "Goodbye!" the moment they hear an "ok". Humans don't. The
7
+ handshake is a short, mandatory state transition that keeps the
8
+ persona from hanging up before the caller does.
9
+
10
+ Flow (per Schegloff & Sacks 1973, *Opening up Closings*):
11
+
12
+ user says "ok" / "alright" / "I should go" / "thanks, that's all"
13
+ β”‚
14
+ β–Ό
15
+ state.phase := 'pre_closing'
16
+ pre_closing_trigger := <the user's token>
17
+ β”‚
18
+ β–Ό
19
+ persona reply directive FORCES a pre-closing token reply:
20
+ - one short softener from facets.pre_closing_tokens
21
+ - no "bye", no "goodbye", no "take care" yet
22
+ β”‚
23
+ β–Ό (next user turn)
24
+ if user says "bye" / "talk later" / "see ya":
25
+ persona reply directive FORCES a mirrored two-part close:
26
+ <softener>, <terminal>
27
+ e.g. "take care, bye"
28
+ else:
29
+ state stays in 'pre_closing'; persona speaks a normal turn.
30
+ """
31
+ from __future__ import annotations
32
+
33
+ import re
34
+ from typing import Optional
35
+
36
+ from .facets import VoiceFacets
37
+
38
+
39
+ # User utterances that start a closing (pre-closing trigger).
40
+ # Kept tight to avoid false positives mid-conversation.
41
+ _PRE_CLOSE_PATTERNS = [
42
+ re.compile(p, re.I) for p in [
43
+ r"^\s*(ok|okay|alright|ok\s+then)\s*[.!?]?\s*$",
44
+ r"\b(i should (go|run|get back)|gotta (go|run)|need to go)\b",
45
+ r"\b(thanks[, ]+(that'?s|that is)\s+(all|it))\b",
46
+ r"\b(talk (to you )?(later|soon)|call (you )?(later|back))\b",
47
+ ]
48
+ ]
49
+
50
+ # Pure terminal β€” "bye"-family tokens that complete the exchange.
51
+ _TERMINAL_PATTERNS = [
52
+ re.compile(p, re.I) for p in [
53
+ r"^\s*(bye|good[\- ]?bye|later|see ya|see you|ciao)\s*[.!?]?\s*$",
54
+ r"\b(bye bye|byebye)\b",
55
+ ]
56
+ ]
57
+
58
+
59
+ def is_pre_closing_trigger(user_text: str) -> bool:
60
+ """True iff the user just began a closing sequence."""
61
+ if not user_text:
62
+ return False
63
+ return any(p.search(user_text) for p in _PRE_CLOSE_PATTERNS)
64
+
65
+
66
+ def is_terminal(user_text: str) -> bool:
67
+ """True iff the user said goodbye β€” the second half of the
68
+ two-phase close. Matched INDEPENDENTLY of the pre-close trigger
69
+ so the state machine can skip straight to terminal if the user
70
+ blurts out "bye!" without a prior "ok"."""
71
+ if not user_text:
72
+ return False
73
+ return any(p.search(user_text) for p in _TERMINAL_PATTERNS)
74
+
75
+
76
+ # ── directive fragments for each phase ───────────────────────────────
77
+
78
+ def pre_closing_directive(facets: VoiceFacets, trigger: Optional[str]) -> str:
79
+ """Directive the composer injects when we're in 'pre_closing'
80
+ but the user hasn't said 'bye' yet."""
81
+ pool = ", ".join(facets.pre_closing_tokens[:4])
82
+ trig = f'"{trigger}"' if trigger else "a closing cue"
83
+ return (
84
+ f"The caller offered a pre-closing ({trig}). "
85
+ "Reciprocate with ONE matching soft pre-closing token β€” "
86
+ f"something from: {pool}. Do NOT say 'goodbye' yet. "
87
+ "Keep the turn to ≀ 7 words and leave the actual goodbye to "
88
+ "the caller. Research: Schegloff & Sacks 1973 β€” the terminal "
89
+ "exchange is the caller's to initiate."
90
+ )
91
+
92
+
93
+ def terminal_directive(facets: VoiceFacets) -> str:
94
+ """Directive for the actual bye-for-bye turn."""
95
+ softeners = ", ".join(facets.pre_closing_tokens[:3])
96
+ closings = ", ".join(facets.closing_tokens[:3])
97
+ return (
98
+ "The caller just said goodbye. Mirror them with a TWO-PART "
99
+ f"close: one softener (e.g. {softeners}) followed by one "
100
+ f"terminal (e.g. {closings}). Never a single bare 'goodbye'. "
101
+ "Keep it to ≀ 6 words. After this turn the call ends."
102
+ )
backend/app/persona_call/config.py ADDED
@@ -0,0 +1,102 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Feature flags + runtime knobs for persona_call.
3
+
4
+ Every knob is an env var with a safe default. The module ships off.
5
+
6
+ Two flags, not one, so "shadow compose" (Β§ rollout P1 in the design)
7
+ can run without changing any outgoing prompt:
8
+
9
+ PERSONA_CALL_ENABLED β€” master. When false, this module is never
10
+ loaded into the app.
11
+ PERSONA_CALL_APPLY β€” when ENABLED=true but APPLY=false, the
12
+ composer runs and its output is logged to
13
+ ``persona_call_directives`` but NOT injected
14
+ as a system message. Lets us collect
15
+ production directives for a week before
16
+ letting them shape turns.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import os
21
+ from dataclasses import dataclass
22
+
23
+
24
+ def _bool_env(name: str, default: bool) -> bool:
25
+ raw = os.getenv(name)
26
+ if raw is None:
27
+ return default
28
+ return raw.strip().lower() in ("1", "true", "yes", "on")
29
+
30
+
31
+ def _int_env(name: str, default: int) -> int:
32
+ raw = os.getenv(name)
33
+ if raw is None:
34
+ return default
35
+ try:
36
+ return int(raw)
37
+ except (TypeError, ValueError):
38
+ return default
39
+
40
+
41
+ def _float_env(name: str, default: float) -> float:
42
+ raw = os.getenv(name)
43
+ if raw is None:
44
+ return default
45
+ try:
46
+ return float(raw)
47
+ except (TypeError, ValueError):
48
+ return default
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class PersonaCallConfig:
53
+ """Resolved at every :func:`load` call β€” cheap, keep fresh for tests."""
54
+
55
+ # Master flag. False = this module is not imported into main.py.
56
+ enabled: bool
57
+ # When True, the composed directive is actually sent to the model.
58
+ # When False, it's logged only (shadow mode).
59
+ apply: bool
60
+
61
+ # Backchannel stream thresholds.
62
+ backchannel_clause_trigger: int # emit after N clause boundaries
63
+ backchannel_min_gap_ms: int # don't emit more often than this
64
+ backchannel_volume_db: float # played at -6 dB by default
65
+
66
+ # Filler emission during tool/LLM wait (Stivers 700 ms trouble line).
67
+ filler_emit_after_ms: int # default 600
68
+
69
+ # Phase machine thresholds.
70
+ how_are_you_late_hour: int # skip "how are you" at or after
71
+ late_night_brevity_hour: int # enter brevity bias at or after
72
+ late_night_brevity_hour_end: int # exit brevity bias at or before
73
+
74
+ # Anti-repetition ledger sizes.
75
+ ack_ledger_window: int
76
+ opener_ledger_window: int
77
+
78
+ # Closing handshake β€” the one enforced override.
79
+ closing_trigger_tokens_extra: str # comma-separated, user-supplied
80
+
81
+ # Reason-for-call fallback turn.
82
+ reason_fallback_turn: int # at this turn, ask if nothing heard
83
+
84
+
85
+ def load() -> PersonaCallConfig:
86
+ return PersonaCallConfig(
87
+ enabled=_bool_env("PERSONA_CALL_ENABLED", False),
88
+ apply=_bool_env("PERSONA_CALL_APPLY", True),
89
+ backchannel_clause_trigger=_int_env("PERSONA_CALL_BC_CLAUSES", 2),
90
+ backchannel_min_gap_ms=_int_env("PERSONA_CALL_BC_MIN_GAP_MS", 1800),
91
+ backchannel_volume_db=_float_env("PERSONA_CALL_BC_VOL_DB", -6.0),
92
+ filler_emit_after_ms=_int_env("PERSONA_CALL_FILLER_AFTER_MS", 600),
93
+ how_are_you_late_hour=_int_env("PERSONA_CALL_HAY_LATE_HOUR", 21),
94
+ late_night_brevity_hour=_int_env("PERSONA_CALL_LATE_HOUR", 22),
95
+ late_night_brevity_hour_end=_int_env("PERSONA_CALL_LATE_HOUR_END", 6),
96
+ ack_ledger_window=_int_env("PERSONA_CALL_ACK_WINDOW", 5),
97
+ opener_ledger_window=_int_env("PERSONA_CALL_OPENER_WINDOW", 3),
98
+ closing_trigger_tokens_extra=os.getenv(
99
+ "PERSONA_CALL_CLOSING_EXTRA_TOKENS", ""
100
+ ),
101
+ reason_fallback_turn=_int_env("PERSONA_CALL_REASON_FALLBACK_TURN", 3),
102
+ )
backend/app/persona_call/context.py ADDED
@@ -0,0 +1,191 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Environmental + caller-context inputs for the per-turn composer.
3
+
4
+ Three kinds of signal, all computed on demand (never stored):
5
+
6
+ (a) Local clock β€” hour, day-of-week, weekend bit. Resolved from the
7
+ caller's ``device_info.tz`` when present, else server tz, else UTC.
8
+ Late-night brevity and morning-energy rules key off this.
9
+
10
+ (b) Recency β€” weeks since this (user, persona) last spoke. Drives the
11
+ "hey, it's been a while" one-shot acknowledgment.
12
+
13
+ (c) Caller-state extraction from STT β€” a tiny regex/keyword classifier
14
+ that runs on every user utterance:
15
+ * mobility (driving / walking / on transit)
16
+ * time pressure (quick question, running late, stepping out)
17
+ * reason-for-call sentence (turn ≀ 3)
18
+
19
+ All inference is local + deterministic. No LLM calls in this module.
20
+ """
21
+ from __future__ import annotations
22
+
23
+ import datetime as _dt
24
+ import re
25
+ from dataclasses import dataclass, field
26
+ from typing import Any, Dict, Optional
27
+
28
+
29
+ # ── local clock ───────────────────────────────────────────────────────
30
+
31
+ @dataclass(frozen=True)
32
+ class CallEnv:
33
+ local_hour: int
34
+ day_of_week: int # 0 Mon … 6 Sun
35
+ is_weekend: bool
36
+ weeks_since_last_call: int # -1 when never
37
+ had_unfinished_topic: bool
38
+ resumed_from_topic: Optional[str] = None
39
+
40
+
41
+ def _resolve_tz(tz_name: Optional[str]):
42
+ """Best-effort tz resolution. Returns a tzinfo or UTC."""
43
+ if not tz_name:
44
+ return _dt.timezone.utc
45
+ try:
46
+ # Python 3.9+ stdlib
47
+ from zoneinfo import ZoneInfo
48
+ return ZoneInfo(tz_name)
49
+ except Exception:
50
+ return _dt.timezone.utc
51
+
52
+
53
+ def compute_env(
54
+ *,
55
+ tz: Optional[str] = None,
56
+ weeks_since_last_call: int = -1,
57
+ resumed_from_topic: Optional[str] = None,
58
+ now: Optional[_dt.datetime] = None,
59
+ ) -> CallEnv:
60
+ """Build a CallEnv from primitive inputs. No DB access here β€”
61
+ ws.py does the session lookups and calls this with the results so
62
+ this function stays cheap and unit-testable."""
63
+ tzinfo = _resolve_tz(tz)
64
+ current = (now or _dt.datetime.now(tz=_dt.timezone.utc)).astimezone(tzinfo)
65
+ return CallEnv(
66
+ local_hour=current.hour,
67
+ day_of_week=current.weekday(),
68
+ is_weekend=current.weekday() >= 5,
69
+ weeks_since_last_call=weeks_since_last_call,
70
+ had_unfinished_topic=bool(resumed_from_topic),
71
+ resumed_from_topic=resumed_from_topic,
72
+ )
73
+
74
+
75
+ # ── caller-state extraction ───────────────────────────────────────────
76
+
77
+ # Deliberately narrow regexes. False positives in this module produce
78
+ # a directive like "the caller is driving β€” keep it to one sentence",
79
+ # which is always a safe fallback; false negatives lose the rule, which
80
+ # is also safe. So tight > comprehensive.
81
+
82
+ _MOBILITY_DRIVING = re.compile(
83
+ r"\b(i[' ]?m|i am|currently|just)?\s*(driving|behind the wheel|on the road|in the car)\b",
84
+ re.I,
85
+ )
86
+ _MOBILITY_WALKING = re.compile(
87
+ r"\b(i[' ]?m|i am)?\s*(walking|out on a walk|going for a walk)\b",
88
+ re.I,
89
+ )
90
+ _MOBILITY_TRANSIT = re.compile(
91
+ r"\b(on (the )?(train|subway|tube|metro|bus)|at the (station|airport)|boarding)\b",
92
+ re.I,
93
+ )
94
+ _TIME_PRESSURE = re.compile(
95
+ r"\b("
96
+ r"quick question|only have a minute|short on time|running late|"
97
+ r"stepping out|have to run|in a rush|in a hurry|gotta be quick"
98
+ r")\b",
99
+ re.I,
100
+ )
101
+
102
+ # Very loose reason-for-call cues for turn ≀ 3. We don't need to
103
+ # extract a perfect paraphrase β€” we just need a yes/no flag so the
104
+ # directive composer can decide whether the phase machine can move
105
+ # out of 'opening'.
106
+ #
107
+ # Intentionally does NOT match "quick question" β€” that is a
108
+ # time-pressure signal, not an articulated reason. The composer reads
109
+ # both signals and will still skip 'how are you' on time pressure
110
+ # alone; we just don't want to prematurely promote phase β†’ topic
111
+ # before the skip rule fires.
112
+ _REASON_CUE = re.compile(
113
+ r"\b("
114
+ r"i('?m| am)?\s+calling (about|to|because|for)|"
115
+ r"i wanted to (talk|ask|check|discuss|tell)|"
116
+ r"the reason i('?m| am) calling|"
117
+ r"i need (to know|help with|your help|a hand)"
118
+ r")\b",
119
+ re.I,
120
+ )
121
+
122
+
123
+ @dataclass
124
+ class CallerSignal:
125
+ driving: bool = False
126
+ walking: bool = False
127
+ transit: bool = False
128
+ time_pressured: bool = False
129
+ reason_expressed: bool = False
130
+ reason_text: str = ""
131
+
132
+ def merge(self, other: "CallerSignal") -> "CallerSignal":
133
+ """Accumulate across multiple user utterances in one call.
134
+ Once a flag is set, it stays set until the phase machine
135
+ resets it."""
136
+ return CallerSignal(
137
+ driving=self.driving or other.driving,
138
+ walking=self.walking or other.walking,
139
+ transit=self.transit or other.transit,
140
+ time_pressured=self.time_pressured or other.time_pressured,
141
+ reason_expressed=self.reason_expressed or other.reason_expressed,
142
+ reason_text=self.reason_text or other.reason_text,
143
+ )
144
+
145
+ def to_dict(self) -> Dict[str, Any]:
146
+ return {
147
+ "driving": self.driving,
148
+ "walking": self.walking,
149
+ "transit": self.transit,
150
+ "time_pressured": self.time_pressured,
151
+ "reason_expressed": self.reason_expressed,
152
+ "reason_text": self.reason_text,
153
+ }
154
+
155
+ @classmethod
156
+ def from_dict(cls, d: Optional[Dict[str, Any]]) -> "CallerSignal":
157
+ d = d or {}
158
+ return cls(
159
+ driving=bool(d.get("driving")),
160
+ walking=bool(d.get("walking")),
161
+ transit=bool(d.get("transit")),
162
+ time_pressured=bool(d.get("time_pressured")),
163
+ reason_expressed=bool(d.get("reason_expressed")),
164
+ reason_text=str(d.get("reason_text") or ""),
165
+ )
166
+
167
+
168
+ def classify_utterance(text: str) -> CallerSignal:
169
+ """Extract a CallerSignal from one user turn. O(len(text)) regex."""
170
+ if not text:
171
+ return CallerSignal()
172
+ sig = CallerSignal(
173
+ driving=bool(_MOBILITY_DRIVING.search(text)),
174
+ walking=bool(_MOBILITY_WALKING.search(text)),
175
+ transit=bool(_MOBILITY_TRANSIT.search(text)),
176
+ time_pressured=bool(_TIME_PRESSURE.search(text)),
177
+ )
178
+ m = _REASON_CUE.search(text)
179
+ if m:
180
+ sig.reason_expressed = True
181
+ # Take the remainder of the sentence as the reason paraphrase
182
+ # β€” a cheap heuristic good enough for the phase machine.
183
+ start = m.end()
184
+ remainder = text[start:].strip()
185
+ end = len(remainder)
186
+ for stop in (".", "?", "!", "\n"):
187
+ idx = remainder.find(stop)
188
+ if idx != -1 and idx < end:
189
+ end = idx
190
+ sig.reason_text = remainder[:end].strip()[:140]
191
+ return sig
backend/app/persona_call/directive.py ADDED
@@ -0,0 +1,365 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Per-turn system-suffix composer.
3
+
4
+ **Core invariant of this whole module** (the product promise):
5
+
6
+ The persona's own system prompt is NEVER touched. `compose()`
7
+ only produces a short suffix that is appended as a SEPARATE
8
+ system message in addition to whatever the /v1/chat/completions
9
+ endpoint already injects for the persona. When the call ends,
10
+ the suffix is gone; the persona's chat behavior is identical to
11
+ what it was before the call started.
12
+
13
+ The suffix is assembled from five small paragraphs. Each paragraph
14
+ is independently testable and individually gated by the current
15
+ phase / time / caller-context, so on an uneventful turn the suffix
16
+ is only ~4 lines long.
17
+
18
+ The five stages, in assembly order:
19
+
20
+ 1. _base_phone_frame persona-neutral phone register
21
+ 2. _phase_frame opening / topic / pre_closing / closed
22
+ 3. _context_frame late-night / driving / resumed-topic
23
+ 4. _repetition_frame ledger-enforced variety
24
+ 5. _latency_frame filler rule for thinking pauses
25
+ """
26
+ from __future__ import annotations
27
+
28
+ from dataclasses import dataclass, field
29
+ from typing import Any, Dict, List, Optional
30
+
31
+ from . import closing, repeat, state as state_mod, store
32
+ from .config import PersonaCallConfig
33
+ from .context import CallEnv, CallerSignal
34
+ from .facets import VoiceFacets
35
+
36
+
37
+ # ── public result type ───────────────────────────────────────────────
38
+
39
+ @dataclass
40
+ class ComposedDirective:
41
+ """Return value of :func:`compose`. Two independently useful fields:
42
+
43
+ * ``system_suffix`` β€” the text we append as an additional system
44
+ message. If :class:`PersonaCallConfig`.apply is False (shadow
45
+ mode), this is logged but not actually sent to the model.
46
+ * ``post_directives`` β€” the list of individual rules we applied,
47
+ one string per rule, in the order composed. Dumped into
48
+ ``persona_call_directives.post_directives`` for observability.
49
+ """
50
+ session_id: str
51
+ phase: str
52
+ turn_index: int
53
+ system_suffix: str
54
+ post_directives: List[str] = field(default_factory=list)
55
+
56
+
57
+ # ── paragraph builders ────────────────────────────────────────────────
58
+
59
+ def _base_phone_frame(facets: VoiceFacets) -> str:
60
+ """Persona-neutral phone register. Same for every persona; the
61
+ persona's identity is NOT asserted here β€” that stays in the
62
+ persona's own system prompt which is already active."""
63
+ return (
64
+ "[PHONE CALL MODE β€” appended only for the duration of this call]\n"
65
+ "You are on a live phone call. Spoken, not typed. "
66
+ "No bullet points, no stage directions (never output '*smiles*' "
67
+ "or similar), no markdown headers. "
68
+ f"Keep replies in the {facets.typical_reply_sentences}-sentence range, "
69
+ f"≀ {facets.sentence_max_words} words per sentence. "
70
+ f"Tone: {facets.tone_line}."
71
+ )
72
+
73
+
74
+ def _phase_frame(
75
+ *,
76
+ phase: str,
77
+ turn_index: int,
78
+ skipped_how_are_you: bool,
79
+ asked_reason_fallback: bool,
80
+ facets: VoiceFacets,
81
+ env: CallEnv,
82
+ caller: CallerSignal,
83
+ pre_closing_trigger: Optional[str],
84
+ cfg: PersonaCallConfig,
85
+ ) -> tuple[str, List[str]]:
86
+ """Return (paragraph, rule_names_applied) for the current phase."""
87
+ rules: List[str] = []
88
+
89
+ if phase == state_mod.OPENING:
90
+ # Turn 0 β€” Schegloff summons-answer.
91
+ if turn_index == 0:
92
+ rules.append("opening.summons_answer")
93
+ return (
94
+ "This is turn 0. Reply with ONE short line of recognition "
95
+ "β€” acknowledge you're there, use the caller's name if you "
96
+ "know it, ≀ 6 words. Do NOT launch into topic.",
97
+ rules,
98
+ )
99
+ # Turn β‰₯ 1 β€” decide on "how are you" exactly once.
100
+ if not skipped_how_are_you and turn_index <= 2:
101
+ # Skip "how are you" if: late hour, caller is pressed, or
102
+ # caller already expressed a reason.
103
+ should_skip = (
104
+ env.local_hour >= cfg.how_are_you_late_hour
105
+ or caller.time_pressured
106
+ or caller.reason_expressed
107
+ )
108
+ if should_skip:
109
+ rules.append("opening.skip_how_are_you")
110
+ return (
111
+ "Skip the 'how are you?' exchange this time β€” the "
112
+ "caller is either pressed, late, or already on-topic. "
113
+ "Move directly to hearing them out.",
114
+ rules,
115
+ )
116
+ rules.append("opening.how_are_you_once")
117
+ return (
118
+ "One brief round of 'how are you?' is appropriate now. "
119
+ "Ask once, let them answer, then pause for their reason "
120
+ "for calling. Do not stack this with another question.",
121
+ rules,
122
+ )
123
+ # Opening past turn 2 with no reason and no fallback asked.
124
+ if (
125
+ not caller.reason_expressed
126
+ and not asked_reason_fallback
127
+ and turn_index >= cfg.reason_fallback_turn
128
+ ):
129
+ rules.append("opening.reason_fallback_ask")
130
+ return (
131
+ "The caller has not said why they're calling yet. "
132
+ "Ask gently, in one sentence: something like "
133
+ "'what's on your mind today?' or 'what's up?'",
134
+ rules,
135
+ )
136
+ rules.append("opening.default")
137
+ return (
138
+ "Still in the opening. Keep replies short and oriented to "
139
+ "the caller's lead.",
140
+ rules,
141
+ )
142
+
143
+ if phase == state_mod.TOPIC:
144
+ rules.append("topic.default")
145
+ return (
146
+ "Mid-call. Be responsive: if the caller asked a question, "
147
+ "answer it before asking one back. Ask at most ONE follow-up "
148
+ "per turn β€” phone calls tolerate fewer questions per turn "
149
+ "than chat does.",
150
+ rules,
151
+ )
152
+
153
+ if phase == state_mod.PRE_CLOSING:
154
+ rules.append("closing.pre_closing_handshake")
155
+ return closing.pre_closing_directive(facets, pre_closing_trigger), rules
156
+
157
+ if phase == state_mod.CLOSED:
158
+ rules.append("closing.terminal")
159
+ return closing.terminal_directive(facets), rules
160
+
161
+ return "", rules
162
+
163
+
164
+ def _context_frame(
165
+ env: CallEnv, caller: CallerSignal, cfg: PersonaCallConfig,
166
+ ) -> tuple[str, List[str]]:
167
+ """Situational rules. Paragraphs only emit when the predicate fires,
168
+ so uneventful turns get no context frame at all."""
169
+ lines: List[str] = []
170
+ rules: List[str] = []
171
+
172
+ # Late-night brevity (decision-fatigue bias).
173
+ late = (
174
+ env.local_hour >= cfg.late_night_brevity_hour
175
+ or env.local_hour <= cfg.late_night_brevity_hour_end
176
+ )
177
+ if late:
178
+ rules.append("context.late_night_brevity")
179
+ lines.append(
180
+ f"Local clock: {env.local_hour:02d}:00 for the caller. "
181
+ "It's late β€” keep replies to one sentence when you can, "
182
+ "offer to continue tomorrow for anything complex."
183
+ )
184
+
185
+ # Driving β€” safety/attention bias.
186
+ if caller.driving:
187
+ rules.append("context.driving")
188
+ lines.append(
189
+ "The caller is DRIVING. One-sentence answers, no multi-part "
190
+ "questions, never require a yes/no decision unless it's "
191
+ "genuinely urgent."
192
+ )
193
+ elif caller.walking or caller.transit:
194
+ rules.append("context.mobile")
195
+ lines.append(
196
+ "The caller is mobile. Keep it brief and well-paced β€” they "
197
+ "may lose signal or attention momentarily."
198
+ )
199
+
200
+ # Weeks-since-last-call resumption (fire at most once per call; the
201
+ # persona_call_state doesn't need a separate flag because this only
202
+ # fires when the topic hasn't been acknowledged yet β€” the composer
203
+ # sees the same state each turn until the persona says something).
204
+ if env.weeks_since_last_call >= 3:
205
+ rules.append("context.resumption")
206
+ lines.append(
207
+ f"It has been β‰ˆ{env.weeks_since_last_call} weeks since this "
208
+ "caller last spoke with you. One natural acknowledgment of "
209
+ "the gap is appropriate β€” do NOT make it the whole turn, "
210
+ "and say it only once this call."
211
+ )
212
+ if env.resumed_from_topic:
213
+ rules.append("context.resume_topic")
214
+ lines.append(
215
+ f"Unfinished topic from a prior call: {env.resumed_from_topic!r}. "
216
+ "Offer to pick it back up if natural, but don't force it."
217
+ )
218
+
219
+ return ("\n".join(lines) if lines else ""), rules
220
+
221
+
222
+ def _repetition_frame(
223
+ state: Dict[str, Any], facets: VoiceFacets,
224
+ ) -> tuple[str, List[str]]:
225
+ text = repeat.forbid_clause(state, facets)
226
+ return (text, ["repeat.ledger"] if text else [])
227
+
228
+
229
+ def _latency_frame(facets: VoiceFacets) -> tuple[str, List[str]]:
230
+ thinking = ", ".join(facets.thinking_tokens[:3])
231
+ return (
232
+ "If you need a beat to think, start your reply with a short "
233
+ f"filler from: {thinking}. Never produce a long silence β€” "
234
+ "callers read silence on the phone as trouble.",
235
+ ["latency.filler_rule"],
236
+ )
237
+
238
+
239
+ # ── public compose() ──────────────────────────────────────────────────
240
+
241
+ def compose(
242
+ *,
243
+ session_id: str,
244
+ persona_id: Optional[str],
245
+ user_text: str,
246
+ env: CallEnv,
247
+ cfg: PersonaCallConfig,
248
+ facets: Optional[VoiceFacets] = None,
249
+ ) -> ComposedDirective:
250
+ """Build the per-turn directive.
251
+
252
+ 1. Advances the phase machine on the user utterance.
253
+ 2. Composes the five paragraphs.
254
+ 3. Persists the directive in shadow mode OR applies it.
255
+ """
256
+ from .facets import for_persona_id
257
+ _facets = facets if facets is not None else for_persona_id(persona_id)
258
+
259
+ # Step 1: phase machine.
260
+ new_state = state_mod.advance_on_user_utterance(
261
+ session_id=session_id,
262
+ user_text=user_text,
263
+ reason_fallback_turn=cfg.reason_fallback_turn,
264
+ )
265
+ phase = str(new_state.get("phase") or state_mod.OPENING)
266
+ turn_index = int(new_state.get("turn_index") or 0)
267
+
268
+ # The caller_context column is already updated by the phase machine,
269
+ # so re-derive the CallerSignal from it for the composer.
270
+ caller = CallerSignal.from_dict(
271
+ new_state.get("caller_context")
272
+ if isinstance(new_state.get("caller_context"), dict) else {}
273
+ )
274
+
275
+ # Step 2: five paragraphs.
276
+ rules_all: List[str] = []
277
+ parts: List[str] = [_base_phone_frame(_facets)]
278
+ rules_all.append("base.phone_frame")
279
+
280
+ phase_para, r = _phase_frame(
281
+ phase=phase,
282
+ turn_index=turn_index - 1, # phase machine incremented already
283
+ skipped_how_are_you=bool(new_state.get("skipped_how_are_you")),
284
+ asked_reason_fallback=bool(new_state.get("asked_reason_fallback")),
285
+ facets=_facets,
286
+ env=env,
287
+ caller=caller,
288
+ pre_closing_trigger=(
289
+ str(new_state.get("pre_closing_trigger") or "")
290
+ if new_state.get("pre_closing_trigger") else None
291
+ ),
292
+ cfg=cfg,
293
+ )
294
+ if phase_para:
295
+ parts.append(phase_para)
296
+ rules_all.extend(r)
297
+
298
+ ctx_para, r = _context_frame(env, caller, cfg)
299
+ if ctx_para:
300
+ parts.append(ctx_para)
301
+ rules_all.extend(r)
302
+
303
+ rep_para, r = _repetition_frame(new_state, _facets)
304
+ if rep_para:
305
+ parts.append(rep_para)
306
+ rules_all.extend(r)
307
+
308
+ lat_para, r = _latency_frame(_facets)
309
+ parts.append(lat_para)
310
+ rules_all.extend(r)
311
+
312
+ system_suffix = "\n\n".join(parts).strip()
313
+
314
+ # Step 3: persist + (maybe) apply.
315
+ store.record_directive(
316
+ session_id=session_id,
317
+ turn_index=turn_index,
318
+ phase=phase,
319
+ applied=cfg.apply,
320
+ system_suffix=system_suffix,
321
+ post_directives=rules_all,
322
+ )
323
+
324
+ # If we asked 'how are you' or the reason-fallback, set the flags
325
+ # so we don't do it again.
326
+ if "opening.how_are_you_once" in rules_all:
327
+ state_mod.mark_how_are_you_decision(session_id, skipped=False)
328
+ if "opening.skip_how_are_you" in rules_all:
329
+ state_mod.mark_how_are_you_decision(session_id, skipped=True)
330
+ if "opening.reason_fallback_ask" in rules_all:
331
+ state_mod.mark_reason_fallback_asked(session_id)
332
+
333
+ return ComposedDirective(
334
+ session_id=session_id,
335
+ phase=phase,
336
+ turn_index=turn_index,
337
+ system_suffix=system_suffix if cfg.apply else "",
338
+ post_directives=rules_all,
339
+ )
340
+
341
+
342
+ def record_persona_reply(
343
+ *,
344
+ session_id: str,
345
+ persona_id: Optional[str],
346
+ reply: str,
347
+ cfg: PersonaCallConfig,
348
+ ) -> None:
349
+ """Called after the LLM reply comes back. Rolls the anti-repetition
350
+ ledger forward so the next turn can forbid the just-used token."""
351
+ from .facets import for_persona_id
352
+ _facets = for_persona_id(persona_id)
353
+ state = store.ensure_state(session_id)
354
+ new_acks, new_openers = repeat.record_reply(
355
+ state,
356
+ reply,
357
+ _facets,
358
+ cfg_ack_window=cfg.ack_ledger_window,
359
+ cfg_opener_window=cfg.opener_ledger_window,
360
+ )
361
+ store.update_state(
362
+ session_id,
363
+ recent_acks=new_acks,
364
+ recent_openers=new_openers,
365
+ )
backend/app/persona_call/facets.py ADDED
@@ -0,0 +1,180 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Per-persona ``voice_facets`` β€” non-destructive voice personalization.
3
+
4
+ Facets live in an OPTIONAL field on the persona manifest
5
+ (``persona.voice_facets``). Any existing persona with no facets gets
6
+ the neutral-English default β€” its chat behavior is completely
7
+ unchanged either way.
8
+
9
+ The facets object drives four things at runtime:
10
+
11
+ * backchannel token pools (``mm``, ``yeah``, ``right`` etc.)
12
+ * pre-closing + closing token pools
13
+ * filler / thinking tokens for the latency hook
14
+ * scalar voice knobs (warmth, pace, formality) interpolated into
15
+ the per-turn system suffix as raw numbers β€” the model reads the
16
+ number, not an adjective. This avoids the same three adjectives
17
+ bleeding into every reply.
18
+
19
+ The *base system prompt* of the persona is never touched. This file
20
+ never rewrites a persona; it only reads the optional facets field.
21
+ """
22
+ from __future__ import annotations
23
+
24
+ from dataclasses import dataclass, field
25
+ from typing import Any, Dict, List, Optional
26
+
27
+
28
+ # ── Default facets β€” applied when a persona ships no voice_facets ───
29
+ #
30
+ # Values chosen from the CA literature + common-sense phone norms.
31
+ # Token pools kept short on purpose: fewer options β†’ easier anti-
32
+ # repetition tracking β†’ cleaner entropy metric.
33
+
34
+ _DEFAULT_ACKS = ["mm", "mm-hm", "yeah", "right", "got it", "okay"]
35
+ _DEFAULT_THINKING = ["hmm", "let me think", "one sec", "hold on"]
36
+ _DEFAULT_PRE_CLOSE = ["okay", "alright", "so", "well"]
37
+ _DEFAULT_CLOSING = ["take care", "talk soon", "bye for now", "later"]
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class VoiceFacets:
42
+ """Immutable snapshot for one persona.
43
+
44
+ All scalar knobs are clamped on load so downstream code can trust
45
+ the range. Token pools are deduplicated + lowercased to keep the
46
+ anti-repetition ledger case-insensitive.
47
+ """
48
+
49
+ warmth: float # 0 clinical … 1 affectionate
50
+ pace: float # 0.25 … 2.0 speech-rate multiplier
51
+ formality: float # 0 casual … 1 formal
52
+ humor: float # 0 none … 1 playful
53
+ self_disclosure: float # 0 reserved … 1 volunteers a lot
54
+ sentence_max_words: int
55
+ typical_reply_sentences: List[int]
56
+ language_register: str
57
+ ack_tokens: List[str]
58
+ thinking_tokens: List[str]
59
+ pre_closing_tokens: List[str]
60
+ closing_tokens: List[str]
61
+ backchannel_rate: float # 0..2, multiplier on default cadence
62
+
63
+ @property
64
+ def tone_line(self) -> str:
65
+ """One-line summary of scalar knobs, ready to drop into the
66
+ per-turn system suffix. Numeric so the model interpolates
67
+ rather than falling back on the same adjective."""
68
+ return (
69
+ f"warmth {self.warmth:.1f} Β· pace {self.pace:.2f} Β· "
70
+ f"formality {self.formality:.1f} Β· humor {self.humor:.1f}"
71
+ )
72
+
73
+
74
+ def _clamp(x: float, lo: float, hi: float) -> float:
75
+ try:
76
+ return max(lo, min(hi, float(x)))
77
+ except (TypeError, ValueError):
78
+ return lo
79
+
80
+
81
+ def _tokens(raw: Any, default: List[str]) -> List[str]:
82
+ """Normalize a token-pool field: list[str], lowercase, de-dup,
83
+ keep stable order. Returns the default if input is empty / bad."""
84
+ if not isinstance(raw, list) or not raw:
85
+ return list(default)
86
+ seen, out = set(), []
87
+ for item in raw:
88
+ if not isinstance(item, str):
89
+ continue
90
+ t = item.strip().lower()
91
+ if t and t not in seen:
92
+ seen.add(t)
93
+ out.append(t)
94
+ return out if out else list(default)
95
+
96
+
97
+ def default_facets() -> VoiceFacets:
98
+ """Neutral English phone-call default. What an unmodified persona
99
+ gets β€” indistinguishable from today's behavior except for the phone
100
+ context itself."""
101
+ return VoiceFacets(
102
+ warmth=0.5,
103
+ pace=1.0,
104
+ formality=0.4,
105
+ humor=0.3,
106
+ self_disclosure=0.3,
107
+ sentence_max_words=22,
108
+ typical_reply_sentences=[1, 2, 3],
109
+ language_register="en-US-casual",
110
+ ack_tokens=list(_DEFAULT_ACKS),
111
+ thinking_tokens=list(_DEFAULT_THINKING),
112
+ pre_closing_tokens=list(_DEFAULT_PRE_CLOSE),
113
+ closing_tokens=list(_DEFAULT_CLOSING),
114
+ backchannel_rate=1.0,
115
+ )
116
+
117
+
118
+ def from_manifest(manifest: Optional[Dict[str, Any]]) -> VoiceFacets:
119
+ """Parse a persona manifest dict into a VoiceFacets.
120
+
121
+ Any missing / malformed field falls back to the default. Safe to
122
+ call with ``None`` or ``{}`` β€” returns :func:`default_facets` in
123
+ those cases. Never raises; a broken facets block must not break a
124
+ call.
125
+ """
126
+ d = default_facets()
127
+ if not isinstance(manifest, dict):
128
+ return d
129
+ raw = manifest.get("voice_facets")
130
+ if not isinstance(raw, dict):
131
+ return d
132
+
133
+ sig = raw.get("signature_tokens") or {}
134
+ if not isinstance(sig, dict):
135
+ sig = {}
136
+
137
+ reply_ns = raw.get("typical_reply_sentences")
138
+ if isinstance(reply_ns, list) and reply_ns:
139
+ reply_ns = [int(n) for n in reply_ns if isinstance(n, (int, float))]
140
+ else:
141
+ reply_ns = list(d.typical_reply_sentences)
142
+
143
+ return VoiceFacets(
144
+ warmth=_clamp(raw.get("warmth", d.warmth), 0.0, 1.0),
145
+ pace=_clamp(raw.get("pace", d.pace), 0.25, 2.0),
146
+ formality=_clamp(raw.get("formality", d.formality), 0.0, 1.0),
147
+ humor=_clamp(raw.get("humor", d.humor), 0.0, 1.0),
148
+ self_disclosure=_clamp(
149
+ raw.get("self_disclosure", d.self_disclosure), 0.0, 1.0
150
+ ),
151
+ sentence_max_words=int(raw.get("sentence_max_words", d.sentence_max_words)),
152
+ typical_reply_sentences=reply_ns,
153
+ language_register=str(raw.get("language_register", d.language_register)),
154
+ ack_tokens=_tokens(sig.get("acknowledgments"), d.ack_tokens),
155
+ thinking_tokens=_tokens(sig.get("thinking"), d.thinking_tokens),
156
+ pre_closing_tokens=_tokens(sig.get("pre_closing"), d.pre_closing_tokens),
157
+ closing_tokens=_tokens(sig.get("closings"), d.closing_tokens),
158
+ backchannel_rate=_clamp(raw.get("backchannel_rate", 1.0), 0.0, 2.0),
159
+ )
160
+
161
+
162
+ # ── Runtime lookup β€” never mutates persona storage ───────────────────
163
+
164
+ def for_persona_id(persona_id: Optional[str]) -> VoiceFacets:
165
+ """Load facets for a persona without touching persona storage.
166
+
167
+ Looks up the persona's manifest (if any) via the existing personas
168
+ module. Any error returns the neutral default β€” persona_call must
169
+ degrade silently when persona storage has not yet been initialized
170
+ (e.g. early tests).
171
+ """
172
+ if not persona_id:
173
+ return default_facets()
174
+ try:
175
+ # Lazy import to avoid any import-time coupling to personas.
176
+ from ..personas.repo import get_persona # type: ignore
177
+ manifest = get_persona(persona_id)
178
+ except Exception:
179
+ return default_facets()
180
+ return from_manifest(manifest)
backend/app/persona_call/latency.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pre-emptive filler scheduler β€” "hmm, one sec…" within the 700 ms
3
+ Stivers trouble threshold.
4
+
5
+ Because the MVP chat path is *non-streaming* (the compat endpoint
6
+ returns 501 on stream=true), the only way to cover long LLM latency
7
+ is to emit a server-originated filler BEFORE the real reply arrives.
8
+
9
+ Usage (from ws.py inside a turn)::
10
+
11
+ async with FillerScheduler(ws=..., facets=facets, cfg=cfg,
12
+ session_id=sid) as sched:
13
+ reply = await turn.run_turn(...)
14
+ # If the turn took longer than cfg.filler_emit_after_ms, the
15
+ # scheduler already emitted one 'assistant.filler' event to the
16
+ # client. Either way, the real reply follows.
17
+
18
+ Per the design doc: filler tokens are persona-specific with a
19
+ sensible universal fallback (covered by ``VoiceFacets.thinking_tokens``,
20
+ whose default pool is ["hmm", "let me think", "one sec", "hold on"]).
21
+ """
22
+ from __future__ import annotations
23
+
24
+ import asyncio
25
+ import random
26
+ import time
27
+ from typing import Any, Callable, Dict, List, Optional
28
+
29
+ from .config import PersonaCallConfig
30
+ from .facets import VoiceFacets
31
+
32
+
33
+ def _now_ms() -> int:
34
+ return int(time.time() * 1000)
35
+
36
+
37
+ class FillerScheduler:
38
+ """Async context-manager that emits at most ONE filler event per
39
+ turn, ``filler_emit_after_ms`` after entering the context. Cancel
40
+ on exit β€” if the LLM returned faster, no event is sent.
41
+
42
+ The event is emitted via the ``send`` callable injected at
43
+ construction, not by calling ``ws.send_text`` directly, so this
44
+ module stays free of fastapi imports and is trivially unit-testable.
45
+ """
46
+
47
+ def __init__(
48
+ self,
49
+ *,
50
+ send: Callable[[Dict[str, Any]], "asyncio.Future"],
51
+ facets: VoiceFacets,
52
+ cfg: PersonaCallConfig,
53
+ session_id: str,
54
+ recent_thinking: Optional[List[str]] = None,
55
+ ) -> None:
56
+ self._send = send
57
+ self._facets = facets
58
+ self._cfg = cfg
59
+ self._sid = session_id
60
+ self._recent = list(recent_thinking or [])
61
+ self._task: Optional[asyncio.Task] = None
62
+ self.emitted: bool = False
63
+ self.emitted_token: Optional[str] = None
64
+
65
+ def _pick_token(self) -> Optional[str]:
66
+ pool = [t for t in self._facets.thinking_tokens if t]
67
+ if not pool:
68
+ return None
69
+ last = (self._recent[-1].lower() if self._recent else "")
70
+ candidates = [t for t in pool if t.lower() != last] or pool
71
+ return random.choice(candidates)
72
+
73
+ async def _fire(self) -> None:
74
+ delay_ms = max(0, int(self._cfg.filler_emit_after_ms))
75
+ try:
76
+ await asyncio.sleep(delay_ms / 1000.0)
77
+ except asyncio.CancelledError:
78
+ return
79
+ token = self._pick_token()
80
+ if not token:
81
+ return
82
+ try:
83
+ await self._send({
84
+ "type": "assistant.filler",
85
+ "payload": {
86
+ "token": token,
87
+ "ts": _now_ms(),
88
+ "session_id": self._sid,
89
+ },
90
+ })
91
+ self.emitted = True
92
+ self.emitted_token = token
93
+ except Exception:
94
+ # Best-effort β€” never let a filler failure break the turn.
95
+ pass
96
+
97
+ async def __aenter__(self) -> "FillerScheduler":
98
+ self._task = asyncio.create_task(self._fire())
99
+ return self
100
+
101
+ async def __aexit__(self, exc_type, exc, tb) -> None:
102
+ if self._task is not None:
103
+ self._task.cancel()
104
+ try:
105
+ await self._task
106
+ except Exception:
107
+ pass
backend/app/persona_call/openings.py ADDED
@@ -0,0 +1,334 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ persona_call.openings β€” "theory of answering" template bank.
3
+
4
+ Adds the deterministic first-turn answer the persona uses when a call
5
+ connects. Backed by conversation-analysis research on phone openings
6
+ (Schegloff 1968, Hopper 1992) and cross-cultural data (Liddicoat
7
+ 2021, ch. 9); augmented with call-center industry practice for the
8
+ formal-register templates.
9
+
10
+ Why a template bank instead of "let the LLM improvise"
11
+ ------------------------------------------------------
12
+ Phone openings are the single most ritualized slot in human
13
+ conversation β€” every culture has a narrow, low-entropy answer
14
+ inventory. A free-form LLM greeting is both slower (latency hurts
15
+ most on turn 1) and less natural (LLMs over-generate: "Hello there!
16
+ What a wonderful day to connect with you…"). A curated bank of
17
+ short templates fires in <5 ms, is persona-aware, is rotated so the
18
+ same caller never hears the same greeting twice in a row, and is
19
+ trivially auditable.
20
+
21
+ Design constraints
22
+ ------------------
23
+ 1. **Additive.** This module is new; nothing else changes. The
24
+ existing ``directive.compose()`` uses it opportunistically in
25
+ the opening phase only.
26
+ 2. **Persona-preserving.** A template is a *seed*, not a mandate:
27
+ the chosen text flows through the persona's own TTS voice and
28
+ the LLM never sees it as an instruction to copy. If a persona
29
+ ships ``voice_facets.signature_tokens.openings``, that override
30
+ wins; templates from this module are used only when the persona
31
+ declines to ship its own.
32
+ 3. **Deterministic under tests.** Selection is seeded by
33
+ (session_id, turn_index) so the same call always picks the same
34
+ template β€” regression tests don't flake.
35
+ 4. **Rotated.** A separate persisted ledger
36
+ (``persona_call_state.recent_openings``) already tracks the
37
+ last few opener tokens the persona used; we extend it to cover
38
+ full template IDs so cross-call repetition is avoided too.
39
+
40
+ Template families (drawn from Hopper 1992 + call-center practice)
41
+ -----------------------------------------------------------------
42
+ - **summons_answer** β€” the minimalist acknowledge-you've-been-summoned
43
+ move: "Hello?", "Yes?", "Mm?" (Schegloff 1968, turn 1).
44
+ - **identification** β€” name-forward answer, the default for most
45
+ non-intimate contexts: "Atlas speaking.", "This is Atlas."
46
+ - **greeting_plus_name** β€” warmer, the default personal-call opener
47
+ in English: "Hi, Atlas here.", "Hey β€” Atlas."
48
+ - **time_of_day** β€” morning/evening variants: "Good evening, Atlas.",
49
+ "Morning β€” Atlas.".
50
+ - **intimate_reopen** β€” for repeat callers (weeks_since_last_call
51
+ < 2): "Hey you.", "Oh hi!", "Good to hear from you.".
52
+ - **call_center_formal** β€” for personas with formality β‰₯ 0.75:
53
+ "Thank you for calling. This is Atlas, how can I help?" and
54
+ variants. The *only* family that includes a how-may-I-help tail.
55
+ - **late_night_brief** β€” after 22:00 local: "Hey…", "Mm, yeah?",
56
+ "Yeah β€” hi.". Short on purpose.
57
+
58
+ Each template carries a ``precond`` callable that decides whether it
59
+ fits the current ``(facets, env, state)``. Selection is a two-pass
60
+ filter+weight: first drop ineligible templates, then weight the
61
+ survivors by persona fit.
62
+ """
63
+
64
+ from __future__ import annotations
65
+
66
+ import hashlib
67
+ import random
68
+ from dataclasses import dataclass, field
69
+ from typing import Callable, Iterable, List, Optional
70
+
71
+ from .context import CallEnv
72
+ from .facets import VoiceFacets
73
+
74
+
75
+ # ── Template record ──────────────────────────────────────────────────
76
+
77
+ @dataclass(frozen=True)
78
+ class OpeningTemplate:
79
+ """One row in the template bank.
80
+
81
+ Attributes
82
+ ----------
83
+ id
84
+ Stable identifier persisted in the anti-repetition ledger.
85
+ Short snake_case.
86
+ family
87
+ One of the seven families in the module docstring. Used by
88
+ selection weighting + tests.
89
+ text
90
+ The template string. ``{name}`` is interpolated with the
91
+ persona's display name (or swallowed if empty). No other
92
+ interpolation β€” keeping templates 100% literal means a
93
+ reviewer can read the bank and know exactly what can ship.
94
+ formality_range
95
+ Inclusive ``(lo, hi)`` on the persona's formality facet.
96
+ Templates outside the persona's range are filtered out.
97
+ precond
98
+ Final predicate run against ``(facets, env)``. Returns True
99
+ iff the template is eligible *right now*. Typical uses:
100
+ late-night variants only after 22:00, intimate variants only
101
+ for recurring callers.
102
+ """
103
+ id: str
104
+ family: str
105
+ text: str
106
+ formality_range: tuple = (0.0, 1.0)
107
+ precond: Callable[[VoiceFacets, CallEnv], bool] = field(
108
+ default=lambda _f, _e: True,
109
+ )
110
+
111
+
112
+ # ── Precondition helpers β€” readable and reusable. ────────────────────
113
+
114
+ def _any_time(_f: VoiceFacets, _e: CallEnv) -> bool:
115
+ return True
116
+
117
+ def _recurring(_f: VoiceFacets, env: CallEnv) -> bool:
118
+ """Fire only when this caller has spoken to this persona recently.
119
+ Guards against a cold-call receiving the intimate family."""
120
+ return 0 <= env.weeks_since_last_call < 2
121
+
122
+ def _late_night(_f: VoiceFacets, env: CallEnv) -> bool:
123
+ return env.local_hour >= 22 or env.local_hour < 6
124
+
125
+ def _morning(_f: VoiceFacets, env: CallEnv) -> bool:
126
+ return 5 <= env.local_hour < 11
127
+
128
+ def _afternoon(_f: VoiceFacets, env: CallEnv) -> bool:
129
+ return 11 <= env.local_hour < 17
130
+
131
+ def _evening(_f: VoiceFacets, env: CallEnv) -> bool:
132
+ return 17 <= env.local_hour < 22
133
+
134
+
135
+ # ── The bank ─────────────────────────────────────────────────────────
136
+ # Deliberately small β€” 23 templates across 7 families. Adding more is
137
+ # cheap but every addition increases the risk of one slipping through
138
+ # review in a voice that doesn't fit the persona. Keep the bar high.
139
+
140
+ OPENING_BANK: tuple = (
141
+ # 1. summons_answer β€” the Schegloff minimum. Any formality.
142
+ OpeningTemplate("sa_hello", "summons_answer", "Hello?"),
143
+ OpeningTemplate("sa_hello_open", "summons_answer", "Hello β€” who's this?"),
144
+ OpeningTemplate("sa_yes", "summons_answer", "Yes?",
145
+ formality_range=(0.0, 0.6)),
146
+ OpeningTemplate("sa_yeah", "summons_answer", "Yeah?",
147
+ formality_range=(0.0, 0.45)),
148
+
149
+ # 2. identification β€” name-forward. Default for 0.3 ≀ formality ≀ 0.8.
150
+ OpeningTemplate("id_name_speaking", "identification",
151
+ "{name} speaking.",
152
+ formality_range=(0.4, 1.0)),
153
+ OpeningTemplate("id_this_is", "identification",
154
+ "This is {name}.",
155
+ formality_range=(0.35, 1.0)),
156
+ OpeningTemplate("id_hello_name", "identification",
157
+ "Hello, this is {name}.",
158
+ formality_range=(0.4, 1.0)),
159
+
160
+ # 3. greeting_plus_name β€” warmer, casual. Default personal-call.
161
+ OpeningTemplate("gn_hi_here", "greeting_plus_name",
162
+ "Hi, {name} here.",
163
+ formality_range=(0.2, 0.75)),
164
+ OpeningTemplate("gn_hey_name", "greeting_plus_name",
165
+ "Hey β€” {name}.",
166
+ formality_range=(0.0, 0.6)),
167
+ OpeningTemplate("gn_hi_hi", "greeting_plus_name",
168
+ "Hi hi.",
169
+ formality_range=(0.0, 0.55)),
170
+
171
+ # 4. time_of_day β€” used only at matching hour, amortised across
172
+ # the bank so it never dominates.
173
+ OpeningTemplate("td_morning", "time_of_day",
174
+ "Morning β€” {name}.",
175
+ precond=_morning),
176
+ OpeningTemplate("td_good_morning", "time_of_day",
177
+ "Good morning, {name} speaking.",
178
+ formality_range=(0.5, 1.0), precond=_morning),
179
+ OpeningTemplate("td_afternoon", "time_of_day",
180
+ "Good afternoon β€” {name}.",
181
+ formality_range=(0.4, 1.0), precond=_afternoon),
182
+ OpeningTemplate("td_evening", "time_of_day",
183
+ "Good evening.",
184
+ formality_range=(0.4, 1.0), precond=_evening),
185
+
186
+ # 5. intimate_reopen β€” recurring callers only.
187
+ OpeningTemplate("int_hey_you", "intimate_reopen",
188
+ "Hey you.",
189
+ formality_range=(0.0, 0.5), precond=_recurring),
190
+ OpeningTemplate("int_oh_hi", "intimate_reopen",
191
+ "Oh, hi!",
192
+ formality_range=(0.0, 0.6), precond=_recurring),
193
+ OpeningTemplate("int_good_to_hear", "intimate_reopen",
194
+ "Hey β€” good to hear from you.",
195
+ formality_range=(0.15, 0.7), precond=_recurring),
196
+
197
+ # 6. call_center_formal β€” high-formality only; the only family
198
+ # that volunteers help.
199
+ OpeningTemplate("cc_thank_you_calling", "call_center_formal",
200
+ "Thank you for calling. This is {name}, how can I help?",
201
+ formality_range=(0.75, 1.0)),
202
+ OpeningTemplate("cc_name_how_help", "call_center_formal",
203
+ "{name} speaking β€” how can I help?",
204
+ formality_range=(0.7, 1.0)),
205
+ OpeningTemplate("cc_standing_by", "call_center_formal",
206
+ "Hello, {name} here β€” what can I do for you?",
207
+ formality_range=(0.65, 1.0)),
208
+
209
+ # 7. late_night_brief β€” just picked up the phone, mid-yawn.
210
+ OpeningTemplate("ln_hey", "late_night_brief",
211
+ "Hey…",
212
+ formality_range=(0.0, 0.55), precond=_late_night),
213
+ OpeningTemplate("ln_yeah_hi", "late_night_brief",
214
+ "Yeah β€” hi.",
215
+ formality_range=(0.0, 0.5), precond=_late_night),
216
+ OpeningTemplate("ln_mm_yeah", "late_night_brief",
217
+ "Mm, yeah?",
218
+ formality_range=(0.0, 0.45), precond=_late_night),
219
+ )
220
+
221
+
222
+ # ── Public API ───────────────────────────────────────────────────────
223
+
224
+ def _interpolate(tpl: str, name: str) -> str:
225
+ """Render ``{name}`` in the template. If the caller has no usable
226
+ display name, swallow the placeholder cleanly so we never ship
227
+ literal ``{name}``."""
228
+ safe = (name or "").strip()
229
+ if not safe:
230
+ # Drop the placeholder + surrounding whitespace around it so
231
+ # "This is {name}." β†’ "This is." wouldn't be ugly. We replace
232
+ # with a near-natural collapse.
233
+ return (
234
+ tpl.replace(", this is {name}.", ".")
235
+ .replace(" this is {name}.", ".")
236
+ .replace(" β€” {name}.", ".")
237
+ .replace("{name} speaking.", "Speaking.")
238
+ .replace("{name} here.", "Here.")
239
+ .replace("{name}", "")
240
+ .strip()
241
+ )
242
+ return tpl.replace("{name}", safe)
243
+
244
+
245
+ def eligible_templates(
246
+ facets: VoiceFacets,
247
+ env: CallEnv,
248
+ *,
249
+ bank: Iterable[OpeningTemplate] = OPENING_BANK,
250
+ forbidden_ids: Iterable[str] = (),
251
+ ) -> List[OpeningTemplate]:
252
+ """Filter the bank to what fits the persona + environment + ledger.
253
+
254
+ Two-pass filter:
255
+ 1. Formality range contains ``facets.formality``.
256
+ 2. ``precond(facets, env)`` returns True.
257
+ 3. ``id`` not in ``forbidden_ids`` (the anti-repetition ledger).
258
+ """
259
+ banned = set(forbidden_ids or ())
260
+ out: List[OpeningTemplate] = []
261
+ for t in bank:
262
+ lo, hi = t.formality_range
263
+ if not (lo <= facets.formality <= hi):
264
+ continue
265
+ if not t.precond(facets, env):
266
+ continue
267
+ if t.id in banned:
268
+ continue
269
+ out.append(t)
270
+ return out
271
+
272
+
273
+ def _seed(session_id: str, turn_index: int) -> int:
274
+ """Stable seed so tests don't flake.
275
+
276
+ Uses blake2b rather than hash() because CPython randomizes
277
+ ``hash()`` between processes (PYTHONHASHSEED), which makes the
278
+ selected template drift under test isolation and is a subtle
279
+ source of flakes.
280
+ """
281
+ h = hashlib.blake2b(
282
+ f"{session_id}:{turn_index}".encode("utf-8"), digest_size=8,
283
+ )
284
+ return int.from_bytes(h.digest(), "big")
285
+
286
+
287
+ def choose(
288
+ facets: VoiceFacets,
289
+ env: CallEnv,
290
+ *,
291
+ session_id: str,
292
+ turn_index: int = 1,
293
+ forbidden_ids: Iterable[str] = (),
294
+ bank: Iterable[OpeningTemplate] = OPENING_BANK,
295
+ ) -> Optional[OpeningTemplate]:
296
+ """Pick one eligible template.
297
+
298
+ Returns the persona's own signature opener when ``facets`` ships a
299
+ custom pool β€” otherwise picks from the filtered bank. Never
300
+ returns an ``OpeningTemplate`` that matches a forbidden id.
301
+
302
+ If no eligible template survives the filter (rare β€” would require
303
+ an unusual formality + time-of-day combination and a ledger full
304
+ of every normally-eligible id), this returns ``None`` and the
305
+ caller should fall back to ``"Hello?"``, which is why the
306
+ summons_answer family has no preconditions.
307
+ """
308
+ pool = eligible_templates(facets, env, bank=bank, forbidden_ids=forbidden_ids)
309
+ if not pool:
310
+ return None
311
+ rnd = random.Random(_seed(session_id, turn_index))
312
+ return rnd.choice(pool)
313
+
314
+
315
+ def render(
316
+ facets: VoiceFacets,
317
+ env: CallEnv,
318
+ *,
319
+ persona_display_name: str = "",
320
+ session_id: str,
321
+ turn_index: int = 1,
322
+ forbidden_ids: Iterable[str] = (),
323
+ ) -> Optional[str]:
324
+ """End-to-end helper: pick + interpolate. Returns None on empty
325
+ pool so the caller can fall back safely."""
326
+ tpl = choose(
327
+ facets, env,
328
+ session_id=session_id,
329
+ turn_index=turn_index,
330
+ forbidden_ids=forbidden_ids,
331
+ )
332
+ if tpl is None:
333
+ return None
334
+ return _interpolate(tpl.text, persona_display_name)
backend/app/persona_call/repeat.py ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Anti-repetition ledger.
3
+
4
+ Three windowed deques per session, all backed by the JSON columns on
5
+ ``persona_call_state``:
6
+
7
+ recent_acks last N acknowledgment tokens the persona used
8
+ recent_openers last N first-words of the persona's turns
9
+ recent_closings last N pre-closing tokens
10
+
11
+ Read API returns the set to FORBID in the next turn's directive; write
12
+ API is called after the reply comes back and extracts the new token via
13
+ a regex-first / LLM-never extractor so it stays cheap.
14
+
15
+ No LLM calls anywhere here β€” the extraction is regex + token-pool
16
+ membership. If the persona's reply starts with a token that isn't in
17
+ its own voice_facets pool, we store the first lowercased word anyway;
18
+ it still anchors the entropy metric.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import re
23
+ from typing import Dict, List, Optional, Set, Tuple
24
+
25
+ from .facets import VoiceFacets
26
+
27
+
28
+ _WORD_BOUNDARY = re.compile(r"^[\s\W]*([a-z][a-z\-']*)", re.I)
29
+
30
+
31
+ def _first_word(reply: str) -> Optional[str]:
32
+ if not reply:
33
+ return None
34
+ m = _WORD_BOUNDARY.match(reply.strip())
35
+ if not m:
36
+ return None
37
+ return m.group(1).lower()
38
+
39
+
40
+ def _roll(deque: List[str], token: str, size: int) -> List[str]:
41
+ out = list(deque or [])
42
+ if token:
43
+ out.append(token)
44
+ if len(out) > size:
45
+ out = out[-size:]
46
+ return out
47
+
48
+
49
+ # ── READ: directive paragraph that forbids recent tokens ─────────────
50
+
51
+ def forbid_clause(
52
+ state: Dict[str, object],
53
+ facets: VoiceFacets,
54
+ *,
55
+ max_len: int = 80,
56
+ ) -> str:
57
+ """Return the one-line directive paragraph about recent tokens.
58
+
59
+ Empty string when there's nothing to forbid yet β€” keeps the
60
+ per-turn suffix short on early turns."""
61
+ acks = state.get("recent_acks") or []
62
+ openers = state.get("recent_openers") or []
63
+ if not acks and not openers:
64
+ return ""
65
+
66
+ dropped: Set[str] = set()
67
+ for t in acks:
68
+ if isinstance(t, str) and t:
69
+ dropped.add(t.lower())
70
+ for t in openers:
71
+ if isinstance(t, str) and t:
72
+ dropped.add(t.lower())
73
+
74
+ if not dropped:
75
+ return ""
76
+
77
+ pool = [t for t in facets.ack_tokens if t.lower() not in dropped]
78
+ fresh_hint = ""
79
+ if pool:
80
+ fresh_hint = f" Prefer a fresh one from: {', '.join(pool[:4])}."
81
+
82
+ forbidden = sorted(dropped)[: max_len // 5] # rough clamp
83
+ return (
84
+ "Avoid repeating your recent openers / acknowledgments: "
85
+ f"{', '.join(forbidden)}." + fresh_hint
86
+ )
87
+
88
+
89
+ # ── WRITE: record the persona's reply into the ledger ────────────────
90
+
91
+ def record_reply(
92
+ state: Dict[str, object],
93
+ reply: str,
94
+ facets: VoiceFacets,
95
+ cfg_ack_window: int,
96
+ cfg_opener_window: int,
97
+ ) -> Tuple[List[str], List[str]]:
98
+ """Roll the ledger forward. Returns the new (recent_acks,
99
+ recent_openers) lists for ``store.update_state``."""
100
+ token = _first_word(reply) or ""
101
+ acks = state.get("recent_acks") or []
102
+ openers = state.get("recent_openers") or []
103
+
104
+ # An ack is anything that appears in the persona's ack pool AND
105
+ # appears in the first three words. That matches how human phone
106
+ # acks actually work ("mm yeah I hear you"): the ack is the front
107
+ # of the turn, not a whole separate utterance.
108
+ front = " ".join((reply or "").lower().strip().split()[:3])
109
+ matched_ack = None
110
+ for pool_token in facets.ack_tokens:
111
+ if pool_token in front:
112
+ matched_ack = pool_token
113
+ break
114
+
115
+ new_acks = _roll(list(acks), matched_ack or "", cfg_ack_window)
116
+ new_openers = _roll(list(openers), token, cfg_opener_window)
117
+ return new_acks, new_openers
118
+
119
+
120
+ def record_closing(
121
+ state: Dict[str, object],
122
+ token: str,
123
+ window: int = 3,
124
+ ) -> List[str]:
125
+ """Roll the closings deque. Called when the closing state machine
126
+ chooses a pre-closing token."""
127
+ prev = state.get("recent_closings") or []
128
+ return _roll(list(prev), token.lower() if token else "", window)
backend/app/persona_call/router.py ADDED
@@ -0,0 +1,224 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Public HTTP surface for persona_call.
3
+
4
+ Five endpoints, all minor β€” this module is primarily called from
5
+ ``voice_call/ws.py`` at runtime. These REST routes are for
6
+ observability + admin tuning + client facets lookup.
7
+
8
+ Mount point in main.py is ONE feature-flagged try/except block.
9
+ With ``PERSONA_CALL_ENABLED=false`` :func:`build_router` returns
10
+ an empty router β€” every path 404s, zero schema is created.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ from typing import Any, Dict, List, Optional
16
+
17
+ from fastapi import (
18
+ APIRouter, Cookie, Depends, Header, HTTPException, Query,
19
+ )
20
+ from pydantic import BaseModel
21
+
22
+ from . import facets as facets_mod
23
+ from . import store
24
+ from .config import PersonaCallConfig
25
+
26
+
27
+ # ── auth passthrough (matches main._scoped_user_or_none) ─────────────
28
+
29
+ def _resolve_user(
30
+ authorization: str,
31
+ homepilot_session: Optional[str],
32
+ ) -> dict:
33
+ from ..users import (
34
+ ensure_users_tables, _validate_token,
35
+ get_or_create_default_user, count_users,
36
+ )
37
+ ensure_users_tables()
38
+ token = ""
39
+ if authorization and authorization.lower().startswith("bearer "):
40
+ token = authorization.split(" ", 1)[1].strip()
41
+ if not token and homepilot_session:
42
+ token = homepilot_session.strip()
43
+ user = _validate_token(token) if token else None
44
+ if user:
45
+ return user
46
+ if count_users() > 1:
47
+ raise HTTPException(status_code=401, detail="Authentication required")
48
+ return get_or_create_default_user()
49
+
50
+
51
+ # ── response models ──────────────────────────────────────────────────
52
+
53
+ class FacetsResp(BaseModel):
54
+ persona_id: Optional[str]
55
+ warmth: float
56
+ pace: float
57
+ formality: float
58
+ humor: float
59
+ self_disclosure: float
60
+ sentence_max_words: int
61
+ typical_reply_sentences: List[int]
62
+ language_register: str
63
+ signature_tokens: Dict[str, List[str]]
64
+ backchannel_rate: float
65
+
66
+
67
+ class StateResp(BaseModel):
68
+ session_id: str
69
+ phase: str
70
+ turn_index: int
71
+ skipped_how_are_you: bool
72
+ asked_reason_fallback: bool
73
+ reason_for_call: str
74
+ pre_closing_trigger: Optional[str]
75
+ recent_acks: List[str]
76
+ recent_openers: List[str]
77
+ recent_closings: List[str]
78
+ caller_context: Dict[str, Any]
79
+
80
+
81
+ class DirectiveResp(BaseModel):
82
+ session_id: str
83
+ turn_index: int
84
+ phase: str
85
+ applied: bool
86
+ system_suffix: str
87
+ post_directives: List[str]
88
+
89
+
90
+ # ── router builder ───────────────────────────────────────────────────
91
+
92
+ def build_router(cfg: PersonaCallConfig) -> APIRouter:
93
+ """Construct the FastAPI router. Empty if ``cfg.enabled`` is
94
+ False so main.py's mount is always a safe no-op."""
95
+ r = APIRouter(prefix="/v1/persona-call", tags=["persona-call"])
96
+ if not cfg.enabled:
97
+ return r
98
+
99
+ @r.get("/facets/{persona_id}", response_model=FacetsResp)
100
+ def get_facets(persona_id: str):
101
+ f = facets_mod.for_persona_id(persona_id)
102
+ return FacetsResp(
103
+ persona_id=persona_id,
104
+ warmth=f.warmth,
105
+ pace=f.pace,
106
+ formality=f.formality,
107
+ humor=f.humor,
108
+ self_disclosure=f.self_disclosure,
109
+ sentence_max_words=f.sentence_max_words,
110
+ typical_reply_sentences=f.typical_reply_sentences,
111
+ language_register=f.language_register,
112
+ signature_tokens={
113
+ "acknowledgments": list(f.ack_tokens),
114
+ "thinking": list(f.thinking_tokens),
115
+ "pre_closing": list(f.pre_closing_tokens),
116
+ "closings": list(f.closing_tokens),
117
+ },
118
+ backchannel_rate=f.backchannel_rate,
119
+ )
120
+
121
+ @r.get("/state/{session_id}", response_model=StateResp)
122
+ def get_state(
123
+ session_id: str,
124
+ authorization: str = Header(default=""),
125
+ homepilot_session: Optional[str] = Cookie(default=None),
126
+ ):
127
+ """Runtime state for the session. Owner-bound via the
128
+ underlying voice_call_sessions row β€” if the session doesn't
129
+ belong to the caller, return 404 (probe-safe)."""
130
+ user = _resolve_user(authorization, homepilot_session)
131
+ # Ownership is enforced by looking up the voice_call_sessions
132
+ # row first. If that is missing or foreign, we 404 before
133
+ # touching persona_call_state at all.
134
+ try:
135
+ from ..voice_call.store import get_session_for_owner
136
+ vc_row = get_session_for_owner(session_id, user["id"])
137
+ except Exception:
138
+ vc_row = None
139
+ if vc_row is None:
140
+ raise HTTPException(status_code=404, detail="Session not found")
141
+ s = store.get_state(session_id)
142
+ if s is None:
143
+ # Session exists but no persona_call state yet β€” return a
144
+ # zero-ed row rather than 404 so clients can poll safely
145
+ # before the first turn.
146
+ return StateResp(
147
+ session_id=session_id,
148
+ phase="opening",
149
+ turn_index=0,
150
+ skipped_how_are_you=False,
151
+ asked_reason_fallback=False,
152
+ reason_for_call="",
153
+ pre_closing_trigger=None,
154
+ recent_acks=[],
155
+ recent_openers=[],
156
+ recent_closings=[],
157
+ caller_context={},
158
+ )
159
+ return StateResp(
160
+ session_id=s["session_id"],
161
+ phase=str(s.get("phase") or "opening"),
162
+ turn_index=int(s.get("turn_index") or 0),
163
+ skipped_how_are_you=bool(s.get("skipped_how_are_you")),
164
+ asked_reason_fallback=bool(s.get("asked_reason_fallback")),
165
+ reason_for_call=str(s.get("reason_for_call") or ""),
166
+ pre_closing_trigger=s.get("pre_closing_trigger"),
167
+ recent_acks=list(s.get("recent_acks") or []),
168
+ recent_openers=list(s.get("recent_openers") or []),
169
+ recent_closings=list(s.get("recent_closings") or []),
170
+ caller_context=dict(s.get("caller_context") or {}),
171
+ )
172
+
173
+ @r.get("/last-directive/{session_id}", response_model=DirectiveResp)
174
+ def get_last_directive(
175
+ session_id: str,
176
+ authorization: str = Header(default=""),
177
+ homepilot_session: Optional[str] = Cookie(default=None),
178
+ ):
179
+ user = _resolve_user(authorization, homepilot_session)
180
+ try:
181
+ from ..voice_call.store import get_session_for_owner
182
+ vc_row = get_session_for_owner(session_id, user["id"])
183
+ except Exception:
184
+ vc_row = None
185
+ if vc_row is None:
186
+ raise HTTPException(status_code=404, detail="Session not found")
187
+ d = store.last_directive(session_id)
188
+ if d is None:
189
+ raise HTTPException(status_code=404, detail="No directive yet")
190
+ return DirectiveResp(
191
+ session_id=d["session_id"],
192
+ turn_index=int(d["turn_index"]),
193
+ phase=str(d["phase"]),
194
+ applied=bool(d["applied"]),
195
+ system_suffix=str(d["system_suffix"] or ""),
196
+ post_directives=list(d["post_directives"] or []),
197
+ )
198
+
199
+ @r.post("/reset/{session_id}")
200
+ def reset_session(
201
+ session_id: str,
202
+ authorization: str = Header(default=""),
203
+ homepilot_session: Optional[str] = Cookie(default=None),
204
+ ):
205
+ """Wipe the anti-repetition ledgers for a session. Owner-bound.
206
+ Used by QA + tests; harmless in production because the phase
207
+ machine just starts fresh from 'opening'."""
208
+ user = _resolve_user(authorization, homepilot_session)
209
+ try:
210
+ from ..voice_call.store import get_session_for_owner
211
+ vc_row = get_session_for_owner(session_id, user["id"])
212
+ except Exception:
213
+ vc_row = None
214
+ if vc_row is None:
215
+ raise HTTPException(status_code=404, detail="Session not found")
216
+ store.update_state(
217
+ session_id,
218
+ recent_acks=[],
219
+ recent_openers=[],
220
+ recent_closings=[],
221
+ )
222
+ return {"ok": True, "session_id": session_id}
223
+
224
+ return r
backend/app/persona_call/state.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Phase state machine.
3
+
4
+ Five phases, deterministic transitions:
5
+
6
+ opening ─(reason expressed OR 'how are you' cycle done)─▢ topic
7
+ topic ─(closing trigger)─────────────────────────────▢ pre_closing
8
+ pre_closing ─(user terminal)──────────────────────────────▢ closed
9
+ pre_closing ─(user recants / new topic)──────────────────▢ topic
10
+ closed (terminal)
11
+
12
+ Phase is the single source of truth for which paragraph
13
+ :mod:`directive` injects into the per-turn system suffix.
14
+ """
15
+ from __future__ import annotations
16
+
17
+ from typing import Dict, Optional
18
+
19
+ from . import closing, context, store
20
+
21
+
22
+ OPENING = "opening"
23
+ TOPIC = "topic"
24
+ PRE_CLOSING = "pre_closing"
25
+ CLOSED = "closed"
26
+
27
+
28
+ def advance_on_user_utterance(
29
+ *,
30
+ session_id: str,
31
+ user_text: str,
32
+ reason_fallback_turn: int,
33
+ ) -> Dict[str, object]:
34
+ """Called by the WS orchestrator BEFORE composing the directive.
35
+
36
+ Updates the phase, turn_index, caller_context, and reason flags
37
+ based on the just-received user utterance. Returns the refreshed
38
+ state row so the composer has the latest view.
39
+ """
40
+ state = store.ensure_state(session_id)
41
+ turn_index = int(state.get("turn_index") or 0)
42
+ phase = state.get("phase") or OPENING
43
+
44
+ # Caller signal extraction β€” accumulate into caller_context.
45
+ prev_signal = context.CallerSignal.from_dict(
46
+ state.get("caller_context") if isinstance(
47
+ state.get("caller_context"), dict) else {}
48
+ )
49
+ fresh = context.classify_utterance(user_text)
50
+ merged = prev_signal.merge(fresh)
51
+
52
+ # Phase transition rules.
53
+ pre_closing_trigger = state.get("pre_closing_trigger")
54
+
55
+ if phase in (OPENING, TOPIC) and closing.is_pre_closing_trigger(user_text):
56
+ phase = PRE_CLOSING
57
+ pre_closing_trigger = user_text.strip()
58
+ elif phase == PRE_CLOSING and closing.is_terminal(user_text):
59
+ phase = CLOSED
60
+ elif phase == OPENING:
61
+ # Move to 'topic' if the user stated a reason, OR if we've
62
+ # already done one "how are you" cycle (2 user turns in
63
+ # opening without a reason).
64
+ if merged.reason_expressed:
65
+ phase = TOPIC
66
+ elif turn_index >= 2:
67
+ # After two turns in 'opening' without a stated reason,
68
+ # the phase-machine forces 'topic' β€” the persona will
69
+ # ask the reason-fallback question via the composer.
70
+ phase = TOPIC
71
+
72
+ # Persist.
73
+ updated = store.update_state(
74
+ session_id,
75
+ phase=phase,
76
+ turn_index=turn_index + 1,
77
+ caller_context=merged.to_dict(),
78
+ pre_closing_trigger=pre_closing_trigger,
79
+ )
80
+ return updated
81
+
82
+
83
+ def mark_how_are_you_decision(session_id: str, *, skipped: bool) -> None:
84
+ """Record that the composer made a 'how are you' decision for this
85
+ session β€” idempotent; the deciding composer path may run multiple
86
+ turns but should only fire once."""
87
+ store.update_state(session_id, skipped_how_are_you=skipped)
88
+
89
+
90
+ def mark_reason_fallback_asked(session_id: str) -> None:
91
+ """Record that the composer already asked the reason-fallback
92
+ question, so it never asks twice."""
93
+ store.update_state(session_id, asked_reason_fallback=True)
94
+
95
+
96
+ def get(session_id: str) -> Optional[Dict[str, object]]:
97
+ """Thin pass-through to the store β€” avoids callers importing
98
+ store directly."""
99
+ return store.get_state(session_id)
backend/app/persona_call/store.py ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ SQLite persistence for persona_call.
3
+
4
+ Two tables, both additive and empty when the feature flag is off:
5
+
6
+ persona_call_state per-session runtime state (phase, turn_index,
7
+ ledgers, caller_context).
8
+
9
+ persona_call_directives shadow-mode audit: every composed directive
10
+ is appended here so P1 (compose but don't
11
+ apply) produces a reviewable trail.
12
+
13
+ Follows the same pattern as backend/app/voice_call/store.py and the
14
+ rest of HomePilot: direct sqlite3, per-call connections, lazy
15
+ ``ensure_schema`` with a process-scoped guard so it's cheap after
16
+ first use.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import json
21
+ import sqlite3
22
+ import time
23
+ from typing import Any, Dict, List, Optional
24
+
25
+
26
+ def _db_path() -> str:
27
+ from ..storage import _get_db_path
28
+ return _get_db_path()
29
+
30
+
31
+ def _conn() -> sqlite3.Connection:
32
+ con = sqlite3.connect(_db_path())
33
+ con.row_factory = sqlite3.Row
34
+ return con
35
+
36
+
37
+ _SCHEMA_READY = False
38
+
39
+
40
+ def ensure_schema() -> None:
41
+ """Create persona_call tables if missing. Safe to call any number
42
+ of times. No-op on subsequent calls within the same process."""
43
+ global _SCHEMA_READY
44
+ if _SCHEMA_READY:
45
+ return
46
+ con = _conn()
47
+ cur = con.cursor()
48
+ cur.executescript(
49
+ """
50
+ CREATE TABLE IF NOT EXISTS persona_call_state (
51
+ session_id TEXT PRIMARY KEY,
52
+ phase TEXT NOT NULL DEFAULT 'opening',
53
+ turn_index INTEGER NOT NULL DEFAULT 0,
54
+ skipped_how_are_you INTEGER NOT NULL DEFAULT 0,
55
+ asked_reason_fallback INTEGER NOT NULL DEFAULT 0,
56
+ reason_for_call TEXT NOT NULL DEFAULT '',
57
+ last_backchannel_ms INTEGER NOT NULL DEFAULT 0,
58
+ recent_acks TEXT NOT NULL DEFAULT '[]',
59
+ recent_openers TEXT NOT NULL DEFAULT '[]',
60
+ recent_closings TEXT NOT NULL DEFAULT '[]',
61
+ caller_context TEXT NOT NULL DEFAULT '{}',
62
+ pre_closing_trigger TEXT,
63
+ created_at INTEGER NOT NULL,
64
+ updated_at INTEGER NOT NULL
65
+ );
66
+
67
+ CREATE TABLE IF NOT EXISTS persona_call_directives (
68
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
69
+ session_id TEXT NOT NULL,
70
+ turn_index INTEGER NOT NULL,
71
+ phase TEXT NOT NULL,
72
+ applied INTEGER NOT NULL DEFAULT 1,
73
+ system_suffix TEXT NOT NULL,
74
+ post_directives TEXT NOT NULL DEFAULT '[]',
75
+ created_at INTEGER NOT NULL
76
+ );
77
+ CREATE INDEX IF NOT EXISTS ix_pcd_session
78
+ ON persona_call_directives (session_id, turn_index);
79
+ """
80
+ )
81
+ con.commit()
82
+ con.close()
83
+ _SCHEMA_READY = True
84
+
85
+
86
+ def _now_ms() -> int:
87
+ return int(time.time() * 1000)
88
+
89
+
90
+ # ── persona_call_state CRUD ───────────────────────────────────────────
91
+
92
+ def get_state(session_id: str) -> Optional[Dict[str, Any]]:
93
+ ensure_schema()
94
+ con = _conn()
95
+ cur = con.cursor()
96
+ cur.execute(
97
+ "SELECT * FROM persona_call_state WHERE session_id = ?",
98
+ (session_id,),
99
+ )
100
+ row = cur.fetchone()
101
+ con.close()
102
+ if row is None:
103
+ return None
104
+ d = {k: row[k] for k in row.keys()}
105
+ # Deserialize JSON columns.
106
+ for col in ("recent_acks", "recent_openers", "recent_closings",
107
+ "caller_context"):
108
+ try:
109
+ d[col] = json.loads(d.get(col) or "[]")
110
+ except Exception:
111
+ d[col] = [] if col != "caller_context" else {}
112
+ return d
113
+
114
+
115
+ def ensure_state(session_id: str) -> Dict[str, Any]:
116
+ """Upsert-shaped: returns the existing row, or creates a fresh one
117
+ in the 'opening' phase if this is the first turn."""
118
+ existing = get_state(session_id)
119
+ if existing is not None:
120
+ return existing
121
+ now = _now_ms()
122
+ con = _conn()
123
+ cur = con.cursor()
124
+ cur.execute(
125
+ """
126
+ INSERT OR IGNORE INTO persona_call_state
127
+ (session_id, created_at, updated_at)
128
+ VALUES (?, ?, ?)
129
+ """,
130
+ (session_id, now, now),
131
+ )
132
+ con.commit()
133
+ con.close()
134
+ return get_state(session_id) # type: ignore[return-value]
135
+
136
+
137
+ def update_state(session_id: str, **changes: Any) -> Dict[str, Any]:
138
+ """Apply a partial update. JSON-typed columns accept Python lists /
139
+ dicts; everything else is a scalar."""
140
+ if not changes:
141
+ return ensure_state(session_id)
142
+
143
+ json_cols = {"recent_acks", "recent_openers", "recent_closings",
144
+ "caller_context"}
145
+ sets, params = [], []
146
+ for col, val in changes.items():
147
+ if col in json_cols:
148
+ val = json.dumps(val, separators=(",", ":"))
149
+ if col in ("skipped_how_are_you", "asked_reason_fallback"):
150
+ val = 1 if val else 0
151
+ sets.append(f"{col} = ?")
152
+ params.append(val)
153
+ sets.append("updated_at = ?")
154
+ params.append(_now_ms())
155
+ params.append(session_id)
156
+
157
+ ensure_state(session_id)
158
+ con = _conn()
159
+ cur = con.cursor()
160
+ cur.execute(
161
+ f"UPDATE persona_call_state SET {', '.join(sets)} WHERE session_id = ?",
162
+ params,
163
+ )
164
+ con.commit()
165
+ con.close()
166
+ return get_state(session_id) # type: ignore[return-value]
167
+
168
+
169
+ # ── persona_call_directives (shadow-mode audit) ───────────────────────
170
+
171
+ def record_directive(
172
+ *,
173
+ session_id: str,
174
+ turn_index: int,
175
+ phase: str,
176
+ applied: bool,
177
+ system_suffix: str,
178
+ post_directives: List[str],
179
+ ) -> None:
180
+ ensure_schema()
181
+ con = _conn()
182
+ cur = con.cursor()
183
+ cur.execute(
184
+ """
185
+ INSERT INTO persona_call_directives
186
+ (session_id, turn_index, phase, applied,
187
+ system_suffix, post_directives, created_at)
188
+ VALUES (?, ?, ?, ?, ?, ?, ?)
189
+ """,
190
+ (
191
+ session_id,
192
+ int(turn_index),
193
+ phase,
194
+ 1 if applied else 0,
195
+ system_suffix,
196
+ json.dumps(post_directives, separators=(",", ":")),
197
+ _now_ms(),
198
+ ),
199
+ )
200
+ con.commit()
201
+ con.close()
202
+
203
+
204
+ def last_directive(session_id: str) -> Optional[Dict[str, Any]]:
205
+ ensure_schema()
206
+ con = _conn()
207
+ cur = con.cursor()
208
+ cur.execute(
209
+ """
210
+ SELECT session_id, turn_index, phase, applied,
211
+ system_suffix, post_directives, created_at
212
+ FROM persona_call_directives
213
+ WHERE session_id = ?
214
+ ORDER BY id DESC
215
+ LIMIT 1
216
+ """,
217
+ (session_id,),
218
+ )
219
+ row = cur.fetchone()
220
+ con.close()
221
+ if row is None:
222
+ return None
223
+ d = {k: row[k] for k in row.keys()}
224
+ try:
225
+ d["post_directives"] = json.loads(d.get("post_directives") or "[]")
226
+ except Exception:
227
+ d["post_directives"] = []
228
+ d["applied"] = bool(d.get("applied"))
229
+ return d
230
+
231
+
232
+ # ── test helper ───────────────────────────────────────────────────────
233
+
234
+ def _reset_for_tests() -> None:
235
+ ensure_schema()
236
+ con = _conn()
237
+ cur = con.cursor()
238
+ cur.execute("DELETE FROM persona_call_directives")
239
+ cur.execute("DELETE FROM persona_call_state")
240
+ con.commit()
241
+ con.close()
backend/app/studio/render_jobs.py CHANGED
@@ -53,6 +53,11 @@ class RenderJob:
53
  duration_sec: float = 0.0
54
  width: int = 0
55
  height: int = 0
 
 
 
 
 
56
 
57
  def to_dict(self) -> Dict[str, Any]:
58
  d = asdict(self)
@@ -115,6 +120,9 @@ def submit_render(
115
  kind: RenderKind,
116
  preset: PlatformPreset,
117
  scenes: List[SceneInput],
 
 
 
118
  ) -> RenderJob:
119
  """Queue a render and return its job record. Worker thread starts immediately."""
120
  job_id = _new_job_id()
@@ -128,6 +136,9 @@ def submit_render(
128
  preset=preset,
129
  status="queued",
130
  output_path=output_path,
 
 
 
131
  )
132
  _store(job)
133
 
@@ -169,6 +180,9 @@ def _run_job(job_id: str, scenes: List[SceneInput]) -> None:
169
  kind=job.kind,
170
  output_path=job.output_path or "",
171
  on_progress=_on_progress,
 
 
 
172
  )
173
  _update(
174
  job_id,
 
53
  duration_sec: float = 0.0
54
  width: int = 0
55
  height: int = 0
56
+ # Render-time audio / subtitle options, captured so /jobs/{id}
57
+ # responses can describe exactly how the file was produced.
58
+ audio_rate: float = 1.0
59
+ audio_pitch: float = 1.0
60
+ subtitles: str = "none"
61
 
62
  def to_dict(self) -> Dict[str, Any]:
63
  d = asdict(self)
 
120
  kind: RenderKind,
121
  preset: PlatformPreset,
122
  scenes: List[SceneInput],
123
+ audio_rate: float = 1.0,
124
+ audio_pitch: float = 1.0,
125
+ subtitles: str = "none",
126
  ) -> RenderJob:
127
  """Queue a render and return its job record. Worker thread starts immediately."""
128
  job_id = _new_job_id()
 
136
  preset=preset,
137
  status="queued",
138
  output_path=output_path,
139
+ audio_rate=audio_rate,
140
+ audio_pitch=audio_pitch,
141
+ subtitles=subtitles,
142
  )
143
  _store(job)
144
 
 
180
  kind=job.kind,
181
  output_path=job.output_path or "",
182
  on_progress=_on_progress,
183
+ audio_rate=job.audio_rate,
184
+ audio_pitch=job.audio_pitch,
185
+ subtitles=job.subtitles,
186
  )
187
  _update(
188
  job_id,
backend/app/studio/render_mp4.py CHANGED
@@ -54,12 +54,17 @@ class SceneInput:
54
  Either ``image_path`` or ``video_path`` should be provided. If both are
55
  set, ``video_path`` wins (it is the higher-fidelity asset). If neither
56
  is set, the scene is rendered as a black frame of ``duration_sec``.
 
 
 
 
57
  """
58
  idx: int
59
  duration_sec: float = 5.0
60
  image_path: Optional[str] = None
61
  video_path: Optional[str] = None
62
  audio_path: Optional[str] = None
 
63
 
64
 
65
  @dataclass
@@ -243,6 +248,119 @@ def _per_scene_args(
243
  return inputs, filters, f"v{scene.idx}"
244
 
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  def _has_audio(path: str) -> bool:
247
  """Probe a media file and return True iff it has at least one audio stream."""
248
  try:
@@ -277,6 +395,9 @@ def render_scenes(
277
  kind: RenderKind,
278
  output_path: str,
279
  on_progress: Optional[Callable[[float], None]] = None,
 
 
 
280
  ) -> RenderResult:
281
  """Render a list of scenes into a single MP4 at ``output_path``.
282
 
@@ -284,6 +405,18 @@ def render_scenes(
284
  then concatenates the intermediates with ffmpeg's concat demuxer. This
285
  is slower than a single filter graph but far more robust against scenes
286
  with very different codecs / resolutions / sample rates / pixel formats.
 
 
 
 
 
 
 
 
 
 
 
 
287
  """
288
  if not ffmpeg_available():
289
  raise RuntimeError(
@@ -294,6 +427,8 @@ def render_scenes(
294
 
295
  width, height, fps = _resolution_for_preset(preset)
296
  encode_args = _encode_args_for_kind(kind, preset)
 
 
297
 
298
  out = Path(output_path)
299
  out.parent.mkdir(parents=True, exist_ok=True)
@@ -325,24 +460,61 @@ def render_scenes(
325
  image_path=image_local,
326
  video_path=video_local,
327
  audio_path=audio_local,
 
328
  )
329
 
330
- inputs, filters, _vlabel = _per_scene_args(
331
  normalized, width, height, fps, scratch
332
  )
333
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
334
  # Audio handling: prefer explicit audio_path; otherwise reuse the
335
  # video's audio track if present; otherwise generate silence so
336
  # every clip has the same stream layout for the concat demuxer.
337
  audio_inputs: list[str] = []
338
  audio_filter: list[str] = []
339
  audio_label = "a0"
 
 
 
 
 
 
 
340
  if audio_local:
341
  audio_inputs += ["-i", audio_local]
342
  # Index 1 is the audio file (after the single video input).
343
- audio_filter.append(f"[1:a]aresample=48000,aformat=channel_layouts=stereo[{audio_label}]")
344
  elif video_local and _has_audio(video_local):
345
- audio_filter.append(f"[0:a]aresample=48000,aformat=channel_layouts=stereo[{audio_label}]")
346
  else:
347
  audio_inputs += [
348
  "-f", "lavfi",
@@ -362,7 +534,7 @@ def render_scenes(
362
  *inputs,
363
  *audio_inputs,
364
  "-filter_complex", filter_complex,
365
- "-map", "[v0]",
366
  "-map", f"[{audio_label}]",
367
  "-r", str(fps),
368
  "-shortest",
@@ -396,6 +568,36 @@ def render_scenes(
396
  ]
397
  _run(concat_cmd)
398
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
399
  if on_progress:
400
  on_progress(100.0)
401
 
 
54
  Either ``image_path`` or ``video_path`` should be provided. If both are
55
  set, ``video_path`` wins (it is the higher-fidelity asset). If neither
56
  is set, the scene is rendered as a black frame of ``duration_sec``.
57
+
58
+ ``narration`` is the source text used to synthesize subtitles when
59
+ the caller passes ``subtitles="burn_in"`` to :func:`render_scenes`.
60
+ It does not affect rendering when subtitles are off.
61
  """
62
  idx: int
63
  duration_sec: float = 5.0
64
  image_path: Optional[str] = None
65
  video_path: Optional[str] = None
66
  audio_path: Optional[str] = None
67
+ narration: str = ""
68
 
69
 
70
  @dataclass
 
248
  return inputs, filters, f"v{scene.idx}"
249
 
250
 
251
+ def _audio_transform_chain(rate: float, pitch: float) -> Optional[str]:
252
+ """Build the ffmpeg audio-filter chain for the chosen rate + pitch.
253
+
254
+ Returns None when both are 1.0 (caller skips the transform stage).
255
+
256
+ Rate is applied with ``atempo``. Pitch uses the industry-standard
257
+ ``asetrate=48000*pitch, atempo=1/pitch`` compound so the tempo is
258
+ preserved while the pitch shifts. When both are non-unit we stack the
259
+ pitch compound first, then an outer atempo for rate.
260
+
261
+ ``atempo`` only accepts 0.5–2.0 per stage; we clamp inputs upstream
262
+ to stay within that range so we never need to chain multiple stages.
263
+ """
264
+ rate = max(0.5, min(2.0, float(rate or 1.0)))
265
+ pitch = max(0.5, min(2.0, float(pitch or 1.0)))
266
+ if rate == 1.0 and pitch == 1.0:
267
+ return None
268
+ parts: list[str] = []
269
+ if pitch != 1.0:
270
+ parts.append(f"asetrate=48000*{pitch:.6f}")
271
+ parts.append(f"atempo={1.0 / pitch:.6f}")
272
+ if rate != 1.0:
273
+ parts.append(f"atempo={rate:.6f}")
274
+ return ",".join(parts)
275
+
276
+
277
+ def _ass_escape(text: str) -> str:
278
+ """Escape a string for embedding inside a libass SRT/ASS cue."""
279
+ return (text or "").replace("\\", "\\\\").replace("{", "\\{").replace("}", "\\}")
280
+
281
+
282
+ def _srt_timestamp(seconds: float) -> str:
283
+ """Convert seconds to an SRT timestamp (HH:MM:SS,mmm)."""
284
+ if seconds < 0:
285
+ seconds = 0.0
286
+ ms = int(round(seconds * 1000))
287
+ hours, ms = divmod(ms, 3_600_000)
288
+ minutes, ms = divmod(ms, 60_000)
289
+ secs, ms = divmod(ms, 1000)
290
+ return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}"
291
+
292
+
293
+ def _split_narration_for_subtitles(text: str, max_chars: int = 42) -> list[str]:
294
+ """Chunk narration into subtitle-sized lines.
295
+
296
+ Subtitles read best at ~42 chars/line (YouTube's auto-caption default).
297
+ We split on sentence boundaries first, then on commas when a chunk is
298
+ still too long, falling back to hard-wrap at word boundaries.
299
+ """
300
+ import re
301
+ if not text:
302
+ return []
303
+ # Sentence split on ., !, ?, or newline.
304
+ sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+|\n+", text) if s.strip()]
305
+ chunks: list[str] = []
306
+ for s in sentences:
307
+ if len(s) <= max_chars:
308
+ chunks.append(s)
309
+ continue
310
+ # Try splitting on commas first.
311
+ sub = [p.strip() for p in s.split(",") if p.strip()]
312
+ if all(len(p) <= max_chars for p in sub):
313
+ chunks.extend(sub)
314
+ continue
315
+ # Fallback: greedy word-wrap.
316
+ cur = ""
317
+ for word in s.split():
318
+ if not cur:
319
+ cur = word
320
+ elif len(cur) + 1 + len(word) <= max_chars:
321
+ cur = f"{cur} {word}"
322
+ else:
323
+ chunks.append(cur)
324
+ cur = word
325
+ if cur:
326
+ chunks.append(cur)
327
+ return chunks
328
+
329
+
330
+ def _write_srt_for_scene(
331
+ narration: str,
332
+ duration_sec: float,
333
+ srt_path: Path,
334
+ *,
335
+ start_offset: float = 0.0,
336
+ ) -> bool:
337
+ """Write an SRT file whose cues span [0, duration_sec] for a single scene.
338
+
339
+ Returns True on success, False when the narration is empty (no SRT
340
+ should be emitted β€” and the caller should not try to burn an empty
341
+ subtitle file, which would fail).
342
+ """
343
+ chunks = _split_narration_for_subtitles(narration)
344
+ if not chunks:
345
+ return False
346
+ dur = max(0.5, float(duration_sec or 5.0))
347
+ # Distribute the duration evenly across chunks. Cap per-cue at 5s so
348
+ # slow voices don't hold a single cue on screen forever.
349
+ per = min(dur / len(chunks), 5.0)
350
+ lines: list[str] = []
351
+ for i, chunk in enumerate(chunks):
352
+ start = start_offset + i * per
353
+ end = min(start_offset + dur, start + per)
354
+ if end <= start:
355
+ end = start + 0.1
356
+ lines.append(str(i + 1))
357
+ lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}")
358
+ lines.append(_ass_escape(chunk))
359
+ lines.append("")
360
+ srt_path.write_text("\n".join(lines), encoding="utf-8")
361
+ return True
362
+
363
+
364
  def _has_audio(path: str) -> bool:
365
  """Probe a media file and return True iff it has at least one audio stream."""
366
  try:
 
395
  kind: RenderKind,
396
  output_path: str,
397
  on_progress: Optional[Callable[[float], None]] = None,
398
+ audio_rate: float = 1.0,
399
+ audio_pitch: float = 1.0,
400
+ subtitles: str = "none",
401
  ) -> RenderResult:
402
  """Render a list of scenes into a single MP4 at ``output_path``.
403
 
 
405
  then concatenates the intermediates with ffmpeg's concat demuxer. This
406
  is slower than a single filter graph but far more robust against scenes
407
  with very different codecs / resolutions / sample rates / pixel formats.
408
+
409
+ Extra knobs:
410
+
411
+ - ``audio_rate`` (0.5–2.0, default 1.0): tempo multiplier for the
412
+ narration/audio track. Applied as ffmpeg ``atempo``.
413
+ - ``audio_pitch`` (0.5–2.0, default 1.0): pitch-shift multiplier.
414
+ Implemented via ``asetrate=48000*pitch, atempo=1/pitch`` so the
415
+ output duration is unchanged (tempo is preserved, pitch shifts).
416
+ - ``subtitles`` (``"burn_in"`` | ``"none"``, default ``"none"``):
417
+ when ``burn_in``, each scene's ``narration`` is chunked into an
418
+ SRT file and burned into the video via libass. A sidecar .srt
419
+ is also written next to the MP4 so editors can re-style.
420
  """
421
  if not ffmpeg_available():
422
  raise RuntimeError(
 
427
 
428
  width, height, fps = _resolution_for_preset(preset)
429
  encode_args = _encode_args_for_kind(kind, preset)
430
+ audio_transform = _audio_transform_chain(audio_rate, audio_pitch)
431
+ burn_subtitles = (subtitles or "none").lower() == "burn_in"
432
 
433
  out = Path(output_path)
434
  out.parent.mkdir(parents=True, exist_ok=True)
 
460
  image_path=image_local,
461
  video_path=video_local,
462
  audio_path=audio_local,
463
+ narration=scene.narration,
464
  )
465
 
466
+ inputs, filters, vlabel = _per_scene_args(
467
  normalized, width, height, fps, scratch
468
  )
469
 
470
+ # If subtitles are burning in, write a per-scene SRT into
471
+ # scratch and append a ``subtitles`` filter to the video chain.
472
+ # ffmpeg's subtitles filter uses libass under the hood. A
473
+ # sidecar .srt is also written next to the final MP4 below.
474
+ if burn_subtitles and scene.narration:
475
+ srt_path = scratch / f"scene_{i:04d}.srt"
476
+ wrote = _write_srt_for_scene(
477
+ scene.narration,
478
+ normalized.duration_sec,
479
+ srt_path,
480
+ )
481
+ if wrote:
482
+ # Escape the path for ffmpeg's filter-arg parser.
483
+ esc = str(srt_path).replace("\\", "/").replace(":", r"\:")
484
+ force_style = (
485
+ "Fontname=DejaVu Sans,Fontsize=28,"
486
+ "PrimaryColour=&H00FFFFFF&,"
487
+ "OutlineColour=&H80000000&,"
488
+ "BorderStyle=3,Outline=2,Shadow=0,MarginV=48"
489
+ )
490
+ # Replace the last video-chain step's output label so
491
+ # the subtitle filter sits at the end of the pipeline.
492
+ new_last = (
493
+ f"[{vlabel}]subtitles='{esc}':"
494
+ f"force_style='{force_style}'[{vlabel}_sub]"
495
+ )
496
+ filters.append(new_last)
497
+ vlabel = f"{vlabel}_sub"
498
+
499
  # Audio handling: prefer explicit audio_path; otherwise reuse the
500
  # video's audio track if present; otherwise generate silence so
501
  # every clip has the same stream layout for the concat demuxer.
502
  audio_inputs: list[str] = []
503
  audio_filter: list[str] = []
504
  audio_label = "a0"
505
+ # Common tail applied to every audio path: resample to 48 kHz
506
+ # stereo + apply the optional rate/pitch transform the caller
507
+ # requested. Consolidating the tail here keeps the three
508
+ # branches below in sync.
509
+ tail = "aresample=48000,aformat=channel_layouts=stereo"
510
+ if audio_transform:
511
+ tail = f"{tail},{audio_transform}"
512
  if audio_local:
513
  audio_inputs += ["-i", audio_local]
514
  # Index 1 is the audio file (after the single video input).
515
+ audio_filter.append(f"[1:a]{tail}[{audio_label}]")
516
  elif video_local and _has_audio(video_local):
517
+ audio_filter.append(f"[0:a]{tail}[{audio_label}]")
518
  else:
519
  audio_inputs += [
520
  "-f", "lavfi",
 
534
  *inputs,
535
  *audio_inputs,
536
  "-filter_complex", filter_complex,
537
+ "-map", f"[{vlabel}]",
538
  "-map", f"[{audio_label}]",
539
  "-r", str(fps),
540
  "-shortest",
 
568
  ]
569
  _run(concat_cmd)
570
 
571
+ # Sidecar SRT: full-video timeline, cue timings chained across
572
+ # scenes. Written whenever burn_subtitles=True, so editors get the
573
+ # same timing they saw burned in. Named <output>.srt.
574
+ if burn_subtitles:
575
+ sidecar = out.with_suffix(".srt")
576
+ running_offset = 0.0
577
+ cue_idx = 1
578
+ lines: list[str] = []
579
+ for s in scenes:
580
+ dur = max(0.5, float(s.duration_sec or 5.0))
581
+ chunks = _split_narration_for_subtitles(s.narration or "")
582
+ if chunks:
583
+ per = min(dur / len(chunks), 5.0)
584
+ for j, chunk in enumerate(chunks):
585
+ start = running_offset + j * per
586
+ end = min(running_offset + dur, start + per)
587
+ if end <= start:
588
+ end = start + 0.1
589
+ lines.append(str(cue_idx))
590
+ lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}")
591
+ lines.append(_ass_escape(chunk))
592
+ lines.append("")
593
+ cue_idx += 1
594
+ running_offset += dur
595
+ try:
596
+ sidecar.write_text("\n".join(lines), encoding="utf-8")
597
+ except OSError:
598
+ # Non-fatal β€” the burned-in subs are still present.
599
+ pass
600
+
601
  if on_progress:
602
  on_progress(100.0)
603
 
backend/app/studio/routes.py CHANGED
@@ -2360,8 +2360,24 @@ class VideoMp4ExportRequest(BaseModel):
2360
  ``kind`` selects the encode profile; the platform preset (canvas
2361
  aspect, fps) is read from the StudioVideo record so the same project
2362
  always renders to the same canvas regardless of the chosen profile.
 
 
 
 
2363
  """
2364
  kind: Literal["mp4_plain", "mp4_youtube"] = "mp4_plain"
 
 
 
 
 
 
 
 
 
 
 
 
2365
 
2366
 
2367
  @router.post("/videos/{video_id}/export/mp4", status_code=202)
@@ -2394,6 +2410,8 @@ def video_export_mp4(
2394
  raise HTTPException(status_code=400, detail="Video has no scenes to render")
2395
 
2396
  # Translate StudioScene records into the renderer's SceneInput shape.
 
 
2397
  inputs = [
2398
  SceneInput(
2399
  idx=i,
@@ -2401,20 +2419,100 @@ def video_export_mp4(
2401
  image_path=s.imageUrl,
2402
  video_path=s.videoUrl,
2403
  audio_path=s.audioUrl,
 
2404
  )
2405
  for i, s in enumerate(scenes)
2406
  ]
2407
 
 
 
 
 
 
 
2408
  job = render_jobs.submit_render(
2409
  user_id=user["id"],
2410
  video_id=video_id,
2411
  kind=req.kind,
2412
  preset=video.platformPreset,
2413
  scenes=inputs,
 
 
 
2414
  )
2415
  return job.to_dict()
2416
 
2417
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2418
  @router.get("/videos/{video_id}/export/jobs/{job_id}")
2419
  def video_export_job_status(
2420
  video_id: str,
 
2360
  ``kind`` selects the encode profile; the platform preset (canvas
2361
  aspect, fps) is read from the StudioVideo record so the same project
2362
  always renders to the same canvas regardless of the chosen profile.
2363
+
2364
+ Audio / subtitle options are additive: the defaults reproduce the
2365
+ original concat-only behavior so the endpoint stays backward
2366
+ compatible for clients that don't send them.
2367
  """
2368
  kind: Literal["mp4_plain", "mp4_youtube"] = "mp4_plain"
2369
+ # Audio rate (tempo) multiplier applied during render, 0.5–2.0. 1.0 is
2370
+ # untouched; <1 slows narration, >1 speeds it up. Duration of the
2371
+ # output clip stretches / shrinks accordingly.
2372
+ audio_rate: float = 1.0
2373
+ # Audio pitch shift, 0.5–2.0. 1.0 is untouched. Implemented with the
2374
+ # industry-standard asetrate + atempo compound so tempo is preserved
2375
+ # while only pitch shifts.
2376
+ audio_pitch: float = 1.0
2377
+ # When "burn_in", scene narration is wrapped into timed SRT cues and
2378
+ # burned into the video via ffmpeg's libass subtitle filter. A sidecar
2379
+ # .srt is always written alongside the MP4 when subtitles != "none".
2380
+ subtitles: Literal["burn_in", "none"] = "none"
2381
 
2382
 
2383
  @router.post("/videos/{video_id}/export/mp4", status_code=202)
 
2410
  raise HTTPException(status_code=400, detail="Video has no scenes to render")
2411
 
2412
  # Translate StudioScene records into the renderer's SceneInput shape.
2413
+ # ``narration`` is the source text for subtitles; it is the same field
2414
+ # the Piper WASM wizard used to synthesize the uploaded audioUrl.
2415
  inputs = [
2416
  SceneInput(
2417
  idx=i,
 
2419
  image_path=s.imageUrl,
2420
  video_path=s.videoUrl,
2421
  audio_path=s.audioUrl,
2422
+ narration=s.narration or "",
2423
  )
2424
  for i, s in enumerate(scenes)
2425
  ]
2426
 
2427
+ # Clamp audio-transform inputs. The renderer itself has additional
2428
+ # defensive clamps, but rejecting absurd values at the API boundary
2429
+ # makes error messages clearer.
2430
+ audio_rate = max(0.5, min(2.0, float(req.audio_rate or 1.0)))
2431
+ audio_pitch = max(0.5, min(2.0, float(req.audio_pitch or 1.0)))
2432
+
2433
  job = render_jobs.submit_render(
2434
  user_id=user["id"],
2435
  video_id=video_id,
2436
  kind=req.kind,
2437
  preset=video.platformPreset,
2438
  scenes=inputs,
2439
+ audio_rate=audio_rate,
2440
+ audio_pitch=audio_pitch,
2441
+ subtitles=req.subtitles,
2442
  )
2443
  return job.to_dict()
2444
 
2445
 
2446
+ # ----------------------------------------------------------------------------
2447
+ # Narration upload
2448
+ # ----------------------------------------------------------------------------
2449
+ # The Creator Studio export wizard synthesizes per-scene narration with the
2450
+ # Piper WASM engine (in-browser), then uploads each WAV here. The server
2451
+ # persists the file under UPLOAD_DIR/studio/narration/<scene_id>.wav and
2452
+ # writes scene.audioUrl. During render the concat pipeline mixes this audio
2453
+ # into the final MP4.
2454
+
2455
+ from fastapi import UploadFile, File
2456
+
2457
+
2458
+ @router.post("/videos/{video_id}/scenes/{scene_id}/narration")
2459
+ async def upload_scene_narration(
2460
+ video_id: str,
2461
+ scene_id: str,
2462
+ file: UploadFile = File(..., description="Audio file (Piper-synthesized WAV)"),
2463
+ authorization: str = Header(default=""),
2464
+ homepilot_session: Optional[str] = Cookie(default=None),
2465
+ ):
2466
+ """Accept a narration audio upload for a scene. Writes ``scene.audioUrl``."""
2467
+ _user = _resolve_user_or_401(authorization, homepilot_session)
2468
+
2469
+ video = get_video(video_id)
2470
+ if not video:
2471
+ raise HTTPException(status_code=404, detail="Video not found")
2472
+ scene = get_scene(scene_id)
2473
+ if not scene or scene.videoId != video_id:
2474
+ raise HTTPException(status_code=404, detail="Scene not found")
2475
+
2476
+ # Piper produces 16-bit PCM WAV. Accept common audio types; the
2477
+ # renderer normalizes anyway.
2478
+ allowed_prefixes = ("audio/", "application/octet-stream")
2479
+ ctype = (file.content_type or "").lower()
2480
+ if ctype and not any(ctype.startswith(p) for p in allowed_prefixes):
2481
+ raise HTTPException(
2482
+ status_code=415,
2483
+ detail=f"Unsupported narration content-type: {file.content_type}",
2484
+ )
2485
+
2486
+ # Cap the upload at 50 MB β€” a minute of 48 kHz 16-bit stereo WAV is
2487
+ # ~11 MB, so this is generous even for long scenes.
2488
+ MAX_BYTES = 50 * 1024 * 1024
2489
+ payload = await file.read(MAX_BYTES + 1)
2490
+ if len(payload) > MAX_BYTES:
2491
+ raise HTTPException(status_code=413, detail="Narration file too large (>50 MB)")
2492
+ if not payload:
2493
+ raise HTTPException(status_code=400, detail="Empty narration file")
2494
+
2495
+ from ..config import UPLOAD_DIR
2496
+ narration_dir = os.path.join(UPLOAD_DIR, "studio", "narration")
2497
+ os.makedirs(narration_dir, exist_ok=True)
2498
+ # Use the scene_id as the filename so re-uploads replace the previous
2499
+ # take cleanly and there is no growing pile of orphan wav files.
2500
+ dest_path = os.path.join(narration_dir, f"{scene_id}.wav")
2501
+ with open(dest_path, "wb") as fh:
2502
+ fh.write(payload)
2503
+
2504
+ # Write back via the existing scene update path so everything
2505
+ # downstream (frontend list, exporter) reads a consistent record.
2506
+ update_scene(scene_id, StudioSceneUpdate(audioUrl=dest_path))
2507
+
2508
+ return {
2509
+ "ok": True,
2510
+ "scene_id": scene_id,
2511
+ "audioUrl": dest_path,
2512
+ "bytes": len(payload),
2513
+ }
2514
+
2515
+
2516
  @router.get("/videos/{video_id}/export/jobs/{job_id}")
2517
  def video_export_job_status(
2518
  video_id: str,
backend/app/voice_call/__init__.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ HomePilot voice-call β€” additive backend module.
3
+
4
+ Scope (MVP M1)
5
+ --------------
6
+ Purely additive feature. With ``VOICE_CALL_ENABLED=false`` (the default)
7
+ this package is never imported into the running app and the DB tables it
8
+ needs are never created β€” dropping the module directory is sufficient to
9
+ remove the feature entirely.
10
+
11
+ Public entry point is :func:`voice_call.router.build_router`; everything
12
+ else is internal. No existing backend module imports from this package.
13
+ """
14
+ from __future__ import annotations
15
+
16
+ __all__ = ["config", "router"]
backend/app/voice_call/barge_in.py ADDED
@@ -0,0 +1,123 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Per-session barge-in token registry.
3
+
4
+ One process-local ``dict`` keyed by ``session_id``; each entry carries
5
+ a single :class:`BargeInToken` representing the currently-active
6
+ assistant turn. The token exposes an ``asyncio.Event`` the streaming
7
+ turn runner polls between chunks β€” when the client sends
8
+ ``user.barge_in`` the WS handler calls :func:`cancel_active` which
9
+ flips the event and the generator exits cleanly on its next poll.
10
+
11
+ The module is deliberately small and state-machine-only:
12
+
13
+ * no HTTP, no DB, no asyncio transport β€” unit-testable in
14
+ milliseconds,
15
+ * ``cancel_active`` refuses cancels for stale ``turn_id`` values so
16
+ a barge-in racing a new turn is a silent no-op,
17
+ * ``cancel`` is idempotent (``asyncio.Event.set()`` is idempotent),
18
+ * ``clear`` wipes a session on WS close so a resume starts fresh.
19
+
20
+ Contract is documented in
21
+ ``docs/analysis/voice-call-streaming-design.md`` Β§ 4.3.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import asyncio
26
+ import threading
27
+ from dataclasses import dataclass, field
28
+ from typing import Dict, Optional
29
+
30
+
31
+ @dataclass
32
+ class BargeInToken:
33
+ """Per-turn cancellation handle.
34
+
35
+ Created by :func:`new_token` when the WS handler starts streaming
36
+ an assistant turn; polled by the streaming runner; flipped by
37
+ :func:`cancel_active` when the user barges in.
38
+ """
39
+
40
+ turn_id: str
41
+ _ev: asyncio.Event = field(default_factory=asyncio.Event)
42
+
43
+ def cancel(self) -> None:
44
+ """Flip the event. Idempotent."""
45
+ self._ev.set()
46
+
47
+ def is_cancelled(self) -> bool:
48
+ """Cheap read the streaming runner calls between chunks."""
49
+ return self._ev.is_set()
50
+
51
+ async def wait(self) -> None:
52
+ """Optional helper for callers that want to ``asyncio.wait``
53
+ on cancel rather than poll."""
54
+ await self._ev.wait()
55
+
56
+
57
+ # ── process-local registry ────────────────────────────────────────────
58
+ # One active token per session. A single lock because the hot path
59
+ # (``cancel_active``) is O(1) β€” no contention worth sharding for.
60
+
61
+ _lock = threading.Lock()
62
+ _active: Dict[str, BargeInToken] = {}
63
+
64
+
65
+ def new_token(session_id: str, turn_id: str) -> BargeInToken:
66
+ """Register ``turn_id`` as the active turn for ``session_id``.
67
+
68
+ Any previous active token for this session is replaced β€” if a
69
+ turn started without the previous one clearing (which would be
70
+ a bug upstream) we fail gracefully rather than leak.
71
+ """
72
+ token = BargeInToken(turn_id=turn_id)
73
+ with _lock:
74
+ _active[session_id] = token
75
+ return token
76
+
77
+
78
+ def cancel_active(session_id: str, turn_id: str) -> bool:
79
+ """Cancel the active turn for ``session_id`` IFF its ``turn_id``
80
+ matches the argument.
81
+
82
+ Returns True on a successful cancel, False if the active turn has
83
+ a different id (stale barge-in) or no turn is active.
84
+ """
85
+ with _lock:
86
+ token = _active.get(session_id)
87
+ if token is None or token.turn_id != turn_id:
88
+ return False
89
+ # Release the lock before setting β€” the event set is itself
90
+ # thread-safe, and keeping the lock narrow keeps the hot path
91
+ # fast under contention.
92
+ token.cancel()
93
+ return True
94
+
95
+
96
+ def get_active(session_id: str) -> Optional[BargeInToken]:
97
+ """Read the currently-active token without mutating.
98
+
99
+ Used by the WS handler to find the active ``turn_id`` when the
100
+ client sends a ``transcript.partial`` (secondary barge-in signal)
101
+ without an explicit ``turn_id``.
102
+ """
103
+ with _lock:
104
+ return _active.get(session_id)
105
+
106
+
107
+ def clear_session(session_id: str) -> None:
108
+ """Drop all tokens for ``session_id`` β€” called on WS close.
109
+
110
+ If a turn was mid-stream we cancel it first so the streaming
111
+ runner exits promptly; the handler is responsible for emitting
112
+ the final envelope.
113
+ """
114
+ with _lock:
115
+ token = _active.pop(session_id, None)
116
+ if token is not None and not token.is_cancelled():
117
+ token.cancel()
118
+
119
+
120
+ def _reset_for_tests() -> None:
121
+ """Test hook β€” wipe the whole registry between cases."""
122
+ with _lock:
123
+ _active.clear()
backend/app/voice_call/config.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ voice_call runtime configuration.
3
+
4
+ Every knob is an environment variable with a safe default. The feature
5
+ ships **off by default** β€” set ``VOICE_CALL_ENABLED=true`` to opt in.
6
+
7
+ Privacy defaults are deliberately conservative per the review note
8
+ ("treat storage/privacy as first-class from day 1"):
9
+ - ``VOICE_CALL_STORE_TRANSCRIPTS`` defaults to ``false``.
10
+ - Metadata persistence (sessions + events) is enabled by default, but
11
+ events only record event *types*, not full audio or raw user text.
12
+ """
13
+ from __future__ import annotations
14
+
15
+ import os
16
+ from dataclasses import dataclass
17
+
18
+
19
+ def _bool_env(name: str, default: bool) -> bool:
20
+ raw = os.getenv(name)
21
+ if raw is None:
22
+ return default
23
+ return raw.strip().lower() in ("1", "true", "yes", "on")
24
+
25
+
26
+ def _int_env(name: str, default: int) -> int:
27
+ raw = os.getenv(name)
28
+ if raw is None:
29
+ return default
30
+ try:
31
+ return int(raw)
32
+ except (TypeError, ValueError):
33
+ return default
34
+
35
+
36
+ @dataclass(frozen=True)
37
+ class VoiceCallConfig:
38
+ """Resolved runtime knobs. Immutable; re-import to refresh in tests."""
39
+
40
+ # Master flag. When false, ``router.build_router()`` returns an empty
41
+ # router and ``store.migrate()`` is a no-op β€” the feature is entirely
42
+ # latent.
43
+ enabled: bool
44
+
45
+ # WebSocket transport for the turn bridge. Off => HTTP-only polling
46
+ # fallback (not yet implemented, so if disabled the feature is
47
+ # effectively off even if ``enabled`` is true).
48
+ websocket_enabled: bool
49
+
50
+ # Provider strategy. Only "internal" is wired at MVP; the string is
51
+ # reserved for future external-realtime adapters.
52
+ provider: str
53
+
54
+ # Policy caps (server-enforced β€” never trust the client).
55
+ max_duration_sec: int
56
+ idle_timeout_sec: int
57
+ resume_window_sec: int
58
+ max_concurrent_per_user: int
59
+ session_create_rate_per_min: int
60
+
61
+ # Privacy. Turn transcripts are NOT persisted by default.
62
+ store_transcripts: bool
63
+ artifact_retention_days: int
64
+
65
+ # Phase 2 + 3 streaming (token-level LLM streaming + barge-in).
66
+ # Defaults off β€” when false, every byte on the wire and every
67
+ # code path taken is identical to today. Independent of
68
+ # ``enabled`` / ``websocket_enabled`` so the streaming path
69
+ # can be dark-shipped alongside unary turns for measurement.
70
+ streaming_enabled: bool
71
+ # Barge-in is an optional sub-feature of streaming; can be
72
+ # disabled independently if the VAD tap is misbehaving on a
73
+ # given deployment.
74
+ barge_in_enabled: bool
75
+
76
+
77
+ def load() -> VoiceCallConfig:
78
+ """Resolve config from env vars every call. Cheap; keep fresh for tests."""
79
+ return VoiceCallConfig(
80
+ enabled=_bool_env("VOICE_CALL_ENABLED", False),
81
+ websocket_enabled=_bool_env("VOICE_CALL_WEBSOCKET_ENABLED", True),
82
+ provider=os.getenv("VOICE_CALL_PROVIDER", "internal").strip() or "internal",
83
+ max_duration_sec=_int_env("VOICE_CALL_MAX_DURATION_SEC", 45 * 60),
84
+ idle_timeout_sec=_int_env("VOICE_CALL_IDLE_TIMEOUT_SEC", 30),
85
+ resume_window_sec=_int_env("VOICE_CALL_RESUME_WINDOW_SEC", 20),
86
+ max_concurrent_per_user=_int_env("VOICE_CALL_MAX_CONCURRENT", 2),
87
+ session_create_rate_per_min=_int_env("VOICE_CALL_CREATE_RATE", 10),
88
+ store_transcripts=_bool_env("VOICE_CALL_STORE_TRANSCRIPTS", False),
89
+ artifact_retention_days=_int_env("VOICE_CALL_ARTIFACT_TTL_DAYS", 30),
90
+ streaming_enabled=_bool_env("VOICE_CALL_STREAMING_ENABLED", False),
91
+ barge_in_enabled=_bool_env("VOICE_CALL_BARGE_IN_ENABLED", True),
92
+ )
backend/app/voice_call/models.py ADDED
@@ -0,0 +1,112 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ voice_call request / response / event schemas.
3
+
4
+ Kept small on purpose: the public surface is half a dozen shapes. If a
5
+ new field is needed, add it optional so the client stays forward-
6
+ compatible with older backends (same rule the rest of HomePilot follows).
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from typing import Any, Dict, List, Literal, Optional
11
+ from pydantic import BaseModel, Field
12
+
13
+
14
+ EntryMode = Literal["header_button", "long_press", "hint_tap", "resume"]
15
+
16
+ SessionStatus = Literal[
17
+ "connecting",
18
+ "live",
19
+ "interrupted", # WS dropped, within resume window
20
+ "ending",
21
+ "ended",
22
+ ]
23
+
24
+
25
+ class DeviceInfo(BaseModel):
26
+ platform: Optional[str] = None # "web" / "ios" / "android" / "desktop"
27
+ app_version: Optional[str] = None
28
+
29
+
30
+ class CreateSessionReq(BaseModel):
31
+ conversation_id: Optional[str] = None
32
+ persona_id: Optional[str] = None
33
+ entry_mode: EntryMode = "header_button"
34
+ device_info: DeviceInfo = Field(default_factory=DeviceInfo)
35
+
36
+
37
+ class Capabilities(BaseModel):
38
+ """What the server promises this session can do. Keep truthful β€” any
39
+ false promise here turns into a UX bug users feel immediately."""
40
+ # At MVP turn-based only. Full duplex (true barge-in) reads as false
41
+ # until we add server-side audio, per the review note.
42
+ interruptions: bool = False
43
+ barge_in: bool = False
44
+ # Assistant replies come as ``transcript.final`` only β€” no partials.
45
+ # This matches the current ``/v1/chat/completions`` behavior which
46
+ # rejects streaming with a 501 (openai_compat_endpoint.py:477).
47
+ transcript_live: bool = False
48
+ # Phase 2/3 opt-in β€” set True on the response when
49
+ # ``VOICE_CALL_STREAMING_ENABLED`` is on. Unaware clients ignore
50
+ # the field and stay on the unary path.
51
+ streaming: bool = False
52
+
53
+
54
+ class CreateSessionResp(BaseModel):
55
+ session_id: str
56
+ ws_url: str
57
+ resume_token: str # short-lived; only accepted by ws handshake
58
+ expires_at: int # epoch ms; WS must open before this
59
+ max_duration_sec: int
60
+ capabilities: Capabilities
61
+
62
+
63
+ class Session(BaseModel):
64
+ """Public session record. Mirrors the DB row without resume_token or
65
+ raw payloads, so an accidental log of this shape never leaks
66
+ credentials."""
67
+ id: str
68
+ user_id: str
69
+ conversation_id: Optional[str] = None
70
+ persona_id: Optional[str] = None
71
+ entry_mode: EntryMode
72
+ status: SessionStatus
73
+ started_at: int
74
+ ended_at: Optional[int] = None
75
+ ended_reason: Optional[str] = None
76
+ duration_sec: Optional[int] = None
77
+
78
+
79
+ class EndSessionResp(BaseModel):
80
+ session_id: str
81
+ status: SessionStatus
82
+ ended_at: int
83
+ ended_reason: str
84
+ duration_sec: int
85
+
86
+
87
+ class HintsResp(BaseModel):
88
+ """Discoverability copy + timings. Server is the source of truth so
89
+ the product team can tune thresholds without a frontend ship."""
90
+ hint_enabled: bool = True
91
+ show_on_mic_tap: bool = True
92
+ auto_hide_ms: int = 1400
93
+ hold_threshold_ms: int = 220
94
+ # Which entry modes this build accepts. Lets us A/B or disable a
95
+ # mode without a client change.
96
+ accepted_entry_modes: List[EntryMode] = Field(
97
+ default_factory=lambda: ["header_button", "long_press", "hint_tap"]
98
+ )
99
+
100
+
101
+ # ── WebSocket event envelope ──────────────────────────────────────────
102
+
103
+ class WsEvent(BaseModel):
104
+ """Every message over the WS β€” both directions β€” wraps into this.
105
+
106
+ ``seq`` is monotonic per session for events the server originates;
107
+ client-originated events don't need it (server ignores it if set).
108
+ """
109
+ type: str
110
+ seq: Optional[int] = None
111
+ ts: Optional[int] = None
112
+ payload: Dict[str, Any] = Field(default_factory=dict)
backend/app/voice_call/policy.py ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Server-enforced policy for voice-call sessions.
3
+
4
+ Never trust the client; every cap here is enforced before a session row
5
+ is inserted or a WebSocket is accepted. Counters live in the DB via
6
+ ``store`` so they survive process restarts and apply across workers.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ from . import store
11
+ from .config import VoiceCallConfig
12
+
13
+
14
+ class PolicyError(Exception):
15
+ """Raised when a caller violates a policy cap. Carries an HTTP status
16
+ and a short machine-readable code so the router can return a clean
17
+ JSON error without leaking internals."""
18
+
19
+ def __init__(self, http_status: int, code: str, detail: str) -> None:
20
+ super().__init__(detail)
21
+ self.http_status = http_status
22
+ self.code = code
23
+ self.detail = detail
24
+
25
+
26
+ def check_session_create(*, user_id: str, cfg: VoiceCallConfig) -> None:
27
+ """Called before :func:`store.create_session`. Raises ``PolicyError``
28
+ on any violation.
29
+
30
+ Checks applied, in order of cheapness:
31
+ 1. ``VOICE_CALL_ENABLED`` β€” feature gate.
32
+ 2. Concurrent session cap (default 2). Stops the "infinite parallel
33
+ calls" abuse vector.
34
+ 3. Rate limit on session creates (default 10 / 60 s). Stops script
35
+ kiddies from hammering ``/sessions``.
36
+ """
37
+ if not cfg.enabled:
38
+ raise PolicyError(
39
+ http_status=404, code="voice_call_disabled",
40
+ detail="Voice call feature is not enabled on this server.",
41
+ )
42
+
43
+ active = store.count_active_sessions_for_user(user_id)
44
+ if active >= cfg.max_concurrent_per_user:
45
+ raise PolicyError(
46
+ http_status=409, code="too_many_active_sessions",
47
+ detail=(
48
+ f"You already have {active} active voice call(s). "
49
+ f"End one before starting a new one."
50
+ ),
51
+ )
52
+
53
+ recent = store.recent_session_creates_for_user(
54
+ user_id, window_sec=60,
55
+ )
56
+ if recent >= cfg.session_create_rate_per_min:
57
+ raise PolicyError(
58
+ http_status=429, code="rate_limited",
59
+ detail=(
60
+ f"Too many voice call starts in the last minute "
61
+ f"(limit {cfg.session_create_rate_per_min})."
62
+ ),
63
+ )
64
+
65
+
66
+ def session_hard_cap_exceeded(session_row: dict, cfg: VoiceCallConfig) -> bool:
67
+ """True if the session has exceeded its hard duration cap.
68
+ The WS orchestrator checks this every heartbeat to force ``ending``."""
69
+ started = int(session_row.get("started_at") or 0)
70
+ if started <= 0:
71
+ return False
72
+ now_ms = int(__import__("time").time() * 1000)
73
+ return (now_ms - started) >= cfg.max_duration_sec * 1000
backend/app/voice_call/router.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Public HTTP + WebSocket surface.
3
+
4
+ All endpoints live under ``/v1/voice-call/*``. The router is built
5
+ dynamically so that with ``VOICE_CALL_ENABLED=false`` it returns an
6
+ empty router β€” no endpoints, no 500s, no accidental exposure.
7
+
8
+ Mount from main.py ONCE, always behind the feature flag::
9
+
10
+ from .voice_call.router import build_router, build_ws_router
11
+ from .voice_call.config import load as _vc_load
12
+ _vc_cfg = _vc_load()
13
+ if _vc_cfg.enabled:
14
+ app.include_router(build_router(_vc_cfg))
15
+ app.include_router(build_ws_router(_vc_cfg))
16
+ """
17
+ from __future__ import annotations
18
+
19
+ import logging
20
+ import os
21
+ from typing import Optional
22
+
23
+ from fastapi import (
24
+ APIRouter, Cookie, Depends, Header, HTTPException,
25
+ Query, WebSocket,
26
+ )
27
+ from fastapi.responses import JSONResponse
28
+
29
+ from . import service, store, ws as ws_mod
30
+ from .config import VoiceCallConfig
31
+ from .models import (
32
+ Capabilities, CreateSessionReq, CreateSessionResp,
33
+ EndSessionResp, HintsResp, Session,
34
+ )
35
+ from .policy import PolicyError
36
+
37
+
38
+ logger = logging.getLogger("voice_call.http")
39
+
40
+
41
+ # ── shared auth resolver (matches main._scoped_user_or_none) ──────────
42
+ # Kept inline to avoid a circular import. Same semantics: single-user
43
+ # installs auto-resolve to the default user; multi-user installs require
44
+ # a bearer token.
45
+
46
+ def _resolve_user(
47
+ authorization: str,
48
+ homepilot_session: Optional[str],
49
+ ) -> dict:
50
+ from ..users import (
51
+ ensure_users_tables, _validate_token,
52
+ get_or_create_default_user, count_users,
53
+ )
54
+ ensure_users_tables()
55
+ token = ""
56
+ if authorization and authorization.lower().startswith("bearer "):
57
+ token = authorization.split(" ", 1)[1].strip()
58
+ if not token and homepilot_session:
59
+ token = homepilot_session.strip()
60
+ user = _validate_token(token) if token else None
61
+ if user:
62
+ return user
63
+ if count_users() > 1:
64
+ raise HTTPException(status_code=401, detail="Authentication required")
65
+ return get_or_create_default_user()
66
+
67
+
68
+ # ── helpers ────────────────────────────────────────────────────────────
69
+
70
+ def _public_session(row: dict) -> Session:
71
+ """Strip the resume_token out before the dict leaves the server."""
72
+ duration = None
73
+ if row.get("ended_at") and row.get("started_at"):
74
+ duration = int((int(row["ended_at"]) - int(row["started_at"])) / 1000)
75
+ return Session(
76
+ id=row["id"],
77
+ user_id=row["user_id"],
78
+ conversation_id=row.get("conversation_id"),
79
+ persona_id=row.get("persona_id"),
80
+ entry_mode=row["entry_mode"],
81
+ status=row["status"],
82
+ started_at=int(row["started_at"]),
83
+ ended_at=int(row["ended_at"]) if row.get("ended_at") else None,
84
+ ended_reason=row.get("ended_reason"),
85
+ duration_sec=duration,
86
+ )
87
+
88
+
89
+ def _ws_url_for(session_id: str) -> str:
90
+ """The WS URL the client should open. Override via env for
91
+ containerized deployments where 127.0.0.1 is wrong."""
92
+ base = (
93
+ os.getenv("VOICE_CALL_PUBLIC_WS_BASE")
94
+ or "ws://127.0.0.1:8000"
95
+ ).rstrip("/")
96
+ return f"{base}/v1/voice-call/ws/{session_id}"
97
+
98
+
99
+ # ── HTTP router ────────────────────────────────────────────────────────
100
+
101
+ def build_router(cfg: VoiceCallConfig) -> APIRouter:
102
+ """Return the REST sub-router. Empty router if feature off."""
103
+ r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"])
104
+ if not cfg.enabled:
105
+ return r # feature off β†’ empty surface
106
+
107
+ @r.post("/sessions", response_model=CreateSessionResp, status_code=201)
108
+ def create_session(
109
+ body: CreateSessionReq,
110
+ authorization: str = Header(default=""),
111
+ homepilot_session: Optional[str] = Cookie(default=None),
112
+ ):
113
+ user = _resolve_user(authorization, homepilot_session)
114
+ try:
115
+ row = service.create_session(
116
+ user_id=user["id"],
117
+ conversation_id=body.conversation_id,
118
+ persona_id=body.persona_id,
119
+ entry_mode=body.entry_mode,
120
+ client_platform=body.device_info.platform,
121
+ app_version=body.device_info.app_version,
122
+ cfg=cfg,
123
+ )
124
+ except PolicyError as exc:
125
+ logger.warning(
126
+ "[call] create_session policy_error user=%s code=%s status=%s",
127
+ user["id"], exc.code, exc.http_status,
128
+ )
129
+ return JSONResponse(
130
+ status_code=exc.http_status,
131
+ content={"ok": False, "code": exc.code, "detail": exc.detail},
132
+ )
133
+ logger.info(
134
+ "[call] create_session ok user=%s sid=%s persona=%s conv=%s",
135
+ user["id"], row["id"], body.persona_id, body.conversation_id,
136
+ )
137
+ # Token is only returned in the create response β€” never again.
138
+ expires_at = int(row["created_at"]) + 15 * 60 * 1000
139
+ return CreateSessionResp(
140
+ session_id=row["id"],
141
+ ws_url=_ws_url_for(row["id"]),
142
+ resume_token=row["resume_token"],
143
+ expires_at=expires_at,
144
+ max_duration_sec=cfg.max_duration_sec,
145
+ capabilities=Capabilities(
146
+ interruptions=False,
147
+ # barge_in reflects the combined streaming + barge-in
148
+ # flag pair β€” it only works when the session actually
149
+ # streams, since there's nothing to cancel in unary.
150
+ barge_in=cfg.streaming_enabled and cfg.barge_in_enabled,
151
+ # transcript_live = streaming assistant partials.
152
+ transcript_live=cfg.streaming_enabled,
153
+ streaming=cfg.streaming_enabled,
154
+ ),
155
+ )
156
+
157
+ @r.get("/sessions/{sid}", response_model=Session)
158
+ def get_session(
159
+ sid: str,
160
+ authorization: str = Header(default=""),
161
+ homepilot_session: Optional[str] = Cookie(default=None),
162
+ ):
163
+ user = _resolve_user(authorization, homepilot_session)
164
+ row = service.get(sid, user["id"])
165
+ if row is None:
166
+ # 404, never 403 β€” probe-safe.
167
+ logger.info("[call] get_session miss user=%s sid=%s", user["id"], sid)
168
+ raise HTTPException(status_code=404, detail="Session not found")
169
+ return _public_session(row)
170
+
171
+ @r.post("/sessions/{sid}/end", response_model=EndSessionResp)
172
+ def end_session_ep(
173
+ sid: str,
174
+ authorization: str = Header(default=""),
175
+ homepilot_session: Optional[str] = Cookie(default=None),
176
+ ):
177
+ user = _resolve_user(authorization, homepilot_session)
178
+ row = service.end_session(sid=sid, user_id=user["id"])
179
+ if row is None:
180
+ logger.info("[call] end_session miss user=%s sid=%s", user["id"], sid)
181
+ raise HTTPException(status_code=404, detail="Session not found")
182
+ logger.info(
183
+ "[call] end_session ok user=%s sid=%s reason=%s",
184
+ user["id"], sid, row.get("ended_reason"),
185
+ )
186
+ public = _public_session(row)
187
+ return EndSessionResp(
188
+ session_id=public.id,
189
+ status=public.status,
190
+ ended_at=public.ended_at or int(row["updated_at"]),
191
+ ended_reason=public.ended_reason or "user_ended",
192
+ duration_sec=public.duration_sec or 0,
193
+ )
194
+
195
+ @r.get("/hints", response_model=HintsResp)
196
+ def hints():
197
+ # Public on purpose β€” the client needs this BEFORE it has a
198
+ # session. Still costs nothing to allow without auth.
199
+ return HintsResp()
200
+
201
+ return r
202
+
203
+
204
+ # ── WS router ──────────────────────────────────────────────────────────
205
+
206
+ def build_ws_router(cfg: VoiceCallConfig) -> APIRouter:
207
+ r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"])
208
+ if not cfg.enabled or not cfg.websocket_enabled:
209
+ return r
210
+
211
+ @r.websocket("/ws/{session_id}")
212
+ async def voice_call_ws(
213
+ websocket: WebSocket,
214
+ session_id: str,
215
+ resume_token: Optional[str] = Query(default=None),
216
+ ):
217
+ # Accept first so we can close with a clean error code if needed.
218
+ await websocket.accept()
219
+ await ws_mod.run_session_ws(
220
+ websocket, session_id, resume_token, cfg,
221
+ )
222
+
223
+ return r
backend/app/voice_call/service.py ADDED
@@ -0,0 +1,145 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Pure-Python state machine for a voice-call session.
3
+
4
+ No HTTP or WebSocket imports here β€” the router/WS layers consume this
5
+ module. That separation lets the unit tests drive the state machine
6
+ end-to-end without spinning up FastAPI.
7
+
8
+ State flow (from the design doc, collapsed for MVP):
9
+
10
+ connecting β†’ live β†’ ending β†’ ended
11
+ β”‚ β”‚
12
+ β”‚ β”œβ”€ interrupted (WS dropped, within resume window)
13
+ β”‚ β”‚ β”‚
14
+ β”‚ β”‚ └─→ live again (on resume)
15
+ β”‚ β”‚ └─→ ended (resume window expired)
16
+ β”‚
17
+ └─ ended (ws never opens)
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import time
22
+ from typing import Any, Dict, Optional
23
+
24
+ from . import store
25
+ from .config import VoiceCallConfig
26
+ from .policy import PolicyError, check_session_create
27
+
28
+
29
+ _TERMINAL_STATES = {"ended"}
30
+ _LIVE_STATES = {"connecting", "live", "interrupted"}
31
+
32
+
33
+ def _now_ms() -> int:
34
+ return int(time.time() * 1000)
35
+
36
+
37
+ # ── create / end ─────────────────────────────────────────────────────
38
+
39
+ def create_session(
40
+ *,
41
+ user_id: str,
42
+ conversation_id: Optional[str],
43
+ persona_id: Optional[str],
44
+ entry_mode: str,
45
+ client_platform: Optional[str],
46
+ app_version: Optional[str],
47
+ cfg: VoiceCallConfig,
48
+ ) -> Dict[str, Any]:
49
+ """Gate through policy, then insert a new session row.
50
+
51
+ The returned dict includes the ``resume_token`` which the router
52
+ will forward to the client. Callers MUST strip this from any
53
+ response that ends up in an observability log.
54
+ """
55
+ check_session_create(user_id=user_id, cfg=cfg)
56
+ row = store.create_session(
57
+ user_id=user_id,
58
+ conversation_id=conversation_id,
59
+ persona_id=persona_id,
60
+ entry_mode=entry_mode,
61
+ client_platform=client_platform,
62
+ app_version=app_version,
63
+ )
64
+ store.append_event(
65
+ session_id=row["id"],
66
+ seq=0,
67
+ event_type="session.created",
68
+ payload={"entry_mode": entry_mode},
69
+ )
70
+ return row
71
+
72
+
73
+ def end_session(
74
+ *,
75
+ sid: str,
76
+ user_id: str,
77
+ reason: str = "user_ended",
78
+ ) -> Optional[Dict[str, Any]]:
79
+ """Mark a session ``ended``. Idempotent: calling twice returns the
80
+ same row on the second call. Returns ``None`` if the session doesn't
81
+ exist OR doesn't belong to ``user_id`` (probe-safe)."""
82
+ row = store.get_session_for_owner(sid, user_id)
83
+ if row is None:
84
+ return None
85
+ if row["status"] in _TERMINAL_STATES:
86
+ return row # already ended; idempotent
87
+ updated = store.update_session_status(sid, "ended", ended_reason=reason)
88
+ if updated is not None:
89
+ store.append_event(
90
+ session_id=sid,
91
+ seq=_next_event_seq(sid),
92
+ event_type="session.ended",
93
+ payload={"reason": reason},
94
+ )
95
+ return updated
96
+
97
+
98
+ def get(sid: str, user_id: str) -> Optional[Dict[str, Any]]:
99
+ """Owner-bound getter. Mirrors :func:`store.get_session_for_owner`
100
+ so callers don't import store directly."""
101
+ return store.get_session_for_owner(sid, user_id)
102
+
103
+
104
+ # ── internal state transitions (called from ws.py) ───────────────────
105
+
106
+ def mark_live(sid: str) -> Optional[Dict[str, Any]]:
107
+ """Transition from ``connecting`` β†’ ``live``. Called when the WS
108
+ opens with a valid resume token."""
109
+ return store.update_session_status(sid, "live")
110
+
111
+
112
+ def mark_interrupted(sid: str) -> Optional[Dict[str, Any]]:
113
+ return store.update_session_status(sid, "interrupted")
114
+
115
+
116
+ def mark_ended_by_timeout(sid: str, reason: str) -> Optional[Dict[str, Any]]:
117
+ return store.update_session_status(sid, "ended", ended_reason=reason)
118
+
119
+
120
+ # ── sequence bookkeeping ─────────────────────────────────────────────
121
+
122
+ # The server emits events with a monotonically-increasing ``seq``. We
123
+ # keep a tiny per-process cache to avoid a SELECT MAX() on every write.
124
+ # The cache is seeded on first use by reading the current max seq for
125
+ # the session β€” so it recovers cleanly across process restarts.
126
+
127
+ from threading import Lock
128
+ _seq_cache: Dict[str, int] = {}
129
+ _seq_lock = Lock()
130
+
131
+
132
+ def _next_event_seq(session_id: str) -> int:
133
+ with _seq_lock:
134
+ if session_id in _seq_cache:
135
+ _seq_cache[session_id] += 1
136
+ return _seq_cache[session_id]
137
+ events = store.list_events(session_id, limit=1000)
138
+ current_max = max((e["seq"] for e in events), default=-1)
139
+ _seq_cache[session_id] = current_max + 1
140
+ return _seq_cache[session_id]
141
+
142
+
143
+ def _reset_for_tests() -> None:
144
+ with _seq_lock:
145
+ _seq_cache.clear()
backend/app/voice_call/store.py ADDED
@@ -0,0 +1,293 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ SQLite access for voice_call.
3
+
4
+ Follows the same pattern as backend/app/storage.py and users.py:
5
+ direct sqlite3 calls, no ORM, per-call connections. Tables are created
6
+ lazily by :func:`ensure_schema` β€” no separate migration file, no side
7
+ effects on import. If the feature is disabled, this module is never
8
+ loaded and no tables are created.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import secrets
14
+ import sqlite3
15
+ import time
16
+ from typing import Any, Dict, List, Optional
17
+
18
+
19
+ # Re-use the same DB path the rest of HomePilot uses.
20
+ def _db_path() -> str:
21
+ from ..storage import _get_db_path # lazy import, avoids cycles
22
+ return _get_db_path()
23
+
24
+
25
+ def _conn() -> sqlite3.Connection:
26
+ con = sqlite3.connect(_db_path())
27
+ con.row_factory = sqlite3.Row
28
+ return con
29
+
30
+
31
+ _SCHEMA_READY = False
32
+
33
+
34
+ def ensure_schema() -> None:
35
+ """Create voice_call tables if they don't exist. Safe to call repeatedly."""
36
+ global _SCHEMA_READY
37
+ if _SCHEMA_READY:
38
+ return
39
+ con = _conn()
40
+ cur = con.cursor()
41
+ cur.executescript(
42
+ """
43
+ CREATE TABLE IF NOT EXISTS voice_call_sessions (
44
+ id TEXT PRIMARY KEY,
45
+ user_id TEXT NOT NULL,
46
+ conversation_id TEXT,
47
+ persona_id TEXT,
48
+ entry_mode TEXT NOT NULL,
49
+ status TEXT NOT NULL,
50
+ resume_token TEXT NOT NULL,
51
+ started_at INTEGER NOT NULL,
52
+ ended_at INTEGER,
53
+ ended_reason TEXT,
54
+ client_platform TEXT,
55
+ app_version TEXT,
56
+ created_at INTEGER NOT NULL,
57
+ updated_at INTEGER NOT NULL
58
+ );
59
+ CREATE INDEX IF NOT EXISTS ix_vcs_user_started
60
+ ON voice_call_sessions (user_id, started_at DESC);
61
+ CREATE INDEX IF NOT EXISTS ix_vcs_user_status
62
+ ON voice_call_sessions (user_id, status);
63
+
64
+ CREATE TABLE IF NOT EXISTS voice_call_events (
65
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
66
+ session_id TEXT NOT NULL,
67
+ seq INTEGER NOT NULL,
68
+ event_type TEXT NOT NULL,
69
+ event_ts INTEGER NOT NULL,
70
+ payload TEXT NOT NULL DEFAULT '{}'
71
+ );
72
+ CREATE INDEX IF NOT EXISTS ix_vce_session_seq
73
+ ON voice_call_events (session_id, seq);
74
+ """
75
+ )
76
+ con.commit()
77
+ con.close()
78
+ _SCHEMA_READY = True
79
+
80
+
81
+ # ── session CRUD ──────────────────────────────────────────────────────
82
+
83
+ def new_session_id() -> str:
84
+ return "vcs_" + secrets.token_urlsafe(16)
85
+
86
+
87
+ def new_resume_token() -> str:
88
+ return secrets.token_urlsafe(24)
89
+
90
+
91
+ def _now_ms() -> int:
92
+ return int(time.time() * 1000)
93
+
94
+
95
+ def create_session(
96
+ *,
97
+ user_id: str,
98
+ conversation_id: Optional[str],
99
+ persona_id: Optional[str],
100
+ entry_mode: str,
101
+ client_platform: Optional[str],
102
+ app_version: Optional[str],
103
+ ) -> Dict[str, Any]:
104
+ """Insert a session row in the ``connecting`` state and return it."""
105
+ ensure_schema()
106
+ sid = new_session_id()
107
+ token = new_resume_token()
108
+ now = _now_ms()
109
+ con = _conn()
110
+ cur = con.cursor()
111
+ cur.execute(
112
+ """
113
+ INSERT INTO voice_call_sessions
114
+ (id, user_id, conversation_id, persona_id, entry_mode, status,
115
+ resume_token, started_at, ended_at, ended_reason,
116
+ client_platform, app_version, created_at, updated_at)
117
+ VALUES
118
+ (?, ?, ?, ?, ?, 'connecting', ?, ?, NULL, NULL, ?, ?, ?, ?)
119
+ """,
120
+ (sid, user_id, conversation_id, persona_id, entry_mode, token,
121
+ now, client_platform, app_version, now, now),
122
+ )
123
+ con.commit()
124
+ con.close()
125
+ return _select_session(sid) # type: ignore[return-value]
126
+
127
+
128
+ def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]:
129
+ d = {k: row[k] for k in row.keys()}
130
+ return d
131
+
132
+
133
+ def _select_session(sid: str) -> Optional[Dict[str, Any]]:
134
+ ensure_schema()
135
+ con = _conn()
136
+ cur = con.cursor()
137
+ cur.execute("SELECT * FROM voice_call_sessions WHERE id = ?", (sid,))
138
+ row = cur.fetchone()
139
+ con.close()
140
+ return _row_to_dict(row) if row else None
141
+
142
+
143
+ def get_session_for_owner(sid: str, user_id: str) -> Optional[Dict[str, Any]]:
144
+ """Fetch a session ONLY if it belongs to ``user_id``.
145
+
146
+ Returns ``None`` for both "not found" and "wrong owner" so callers
147
+ cannot probe session ids they don't own. Matches the pattern used
148
+ by studio render jobs.
149
+ """
150
+ row = _select_session(sid)
151
+ if row and row["user_id"] == user_id:
152
+ return row
153
+ return None
154
+
155
+
156
+ def count_active_sessions_for_user(user_id: str) -> int:
157
+ """Count sessions that are still 'live' or 'connecting' or 'interrupted'."""
158
+ ensure_schema()
159
+ con = _conn()
160
+ cur = con.cursor()
161
+ cur.execute(
162
+ """
163
+ SELECT COUNT(*) AS n FROM voice_call_sessions
164
+ WHERE user_id = ?
165
+ AND status IN ('connecting', 'live', 'interrupted')
166
+ """,
167
+ (user_id,),
168
+ )
169
+ n = int(cur.fetchone()["n"])
170
+ con.close()
171
+ return n
172
+
173
+
174
+ def recent_session_creates_for_user(user_id: str, window_sec: int) -> int:
175
+ """Session creates by ``user_id`` in the last ``window_sec`` seconds."""
176
+ ensure_schema()
177
+ con = _conn()
178
+ cur = con.cursor()
179
+ since = _now_ms() - int(window_sec * 1000)
180
+ cur.execute(
181
+ """
182
+ SELECT COUNT(*) AS n FROM voice_call_sessions
183
+ WHERE user_id = ? AND created_at >= ?
184
+ """,
185
+ (user_id, since),
186
+ )
187
+ n = int(cur.fetchone()["n"])
188
+ con.close()
189
+ return n
190
+
191
+
192
+ def update_session_status(
193
+ sid: str,
194
+ status: str,
195
+ *,
196
+ ended_reason: Optional[str] = None,
197
+ ) -> Optional[Dict[str, Any]]:
198
+ ensure_schema()
199
+ now = _now_ms()
200
+ con = _conn()
201
+ cur = con.cursor()
202
+ if status == "ended":
203
+ cur.execute(
204
+ """
205
+ UPDATE voice_call_sessions
206
+ SET status = 'ended', ended_at = ?, ended_reason = ?, updated_at = ?
207
+ WHERE id = ?
208
+ """,
209
+ (now, ended_reason or "user_ended", now, sid),
210
+ )
211
+ else:
212
+ cur.execute(
213
+ """
214
+ UPDATE voice_call_sessions
215
+ SET status = ?, updated_at = ?
216
+ WHERE id = ?
217
+ """,
218
+ (status, now, sid),
219
+ )
220
+ con.commit()
221
+ con.close()
222
+ return _select_session(sid)
223
+
224
+
225
+ # ── events ────────────────────────────────────────────────────────────
226
+
227
+ def append_event(
228
+ *,
229
+ session_id: str,
230
+ seq: int,
231
+ event_type: str,
232
+ payload: Optional[Dict[str, Any]] = None,
233
+ ) -> None:
234
+ """Append a structured event row. Payload defaults to {} β€” DO NOT
235
+ pass raw audio or full user text here unless privacy flag allows it."""
236
+ ensure_schema()
237
+ con = _conn()
238
+ cur = con.cursor()
239
+ cur.execute(
240
+ """
241
+ INSERT INTO voice_call_events (session_id, seq, event_type, event_ts, payload)
242
+ VALUES (?, ?, ?, ?, ?)
243
+ """,
244
+ (
245
+ session_id,
246
+ int(seq),
247
+ event_type,
248
+ _now_ms(),
249
+ json.dumps(payload or {}, separators=(",", ":")),
250
+ ),
251
+ )
252
+ con.commit()
253
+ con.close()
254
+
255
+
256
+ def list_events(session_id: str, limit: int = 500) -> List[Dict[str, Any]]:
257
+ ensure_schema()
258
+ con = _conn()
259
+ cur = con.cursor()
260
+ cur.execute(
261
+ """
262
+ SELECT session_id, seq, event_type, event_ts, payload
263
+ FROM voice_call_events
264
+ WHERE session_id = ?
265
+ ORDER BY seq ASC
266
+ LIMIT ?
267
+ """,
268
+ (session_id, int(limit)),
269
+ )
270
+ rows = cur.fetchall()
271
+ con.close()
272
+ out = []
273
+ for r in rows:
274
+ d = _row_to_dict(r)
275
+ try:
276
+ d["payload"] = json.loads(d.get("payload") or "{}")
277
+ except Exception:
278
+ d["payload"] = {}
279
+ out.append(d)
280
+ return out
281
+
282
+
283
+ # ── test helper ───────────────────────────────────────────────────────
284
+
285
+ def _reset_for_tests() -> None:
286
+ """DELETE every voice_call row. Only used by pytest fixtures."""
287
+ ensure_schema()
288
+ con = _conn()
289
+ cur = con.cursor()
290
+ cur.execute("DELETE FROM voice_call_events")
291
+ cur.execute("DELETE FROM voice_call_sessions")
292
+ con.commit()
293
+ con.close()
backend/app/voice_call/turn.py ADDED
@@ -0,0 +1,125 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Turn bridge β€” send a ``transcript.final`` from the client to the
3
+ existing ``/v1/chat/completions`` endpoint and get back an assistant
4
+ reply (also final, no deltas).
5
+
6
+ Implementation note
7
+ -------------------
8
+ The review confirmed the compat endpoint currently returns 501 on
9
+ ``stream=true`` (openai_compat_endpoint.py:477), so MVP M1 is final-
10
+ only. We invoke the endpoint via an internal loopback HTTP call; that
11
+ buys:
12
+
13
+ * zero import-time coupling to the chat module,
14
+ * the same auth / memory / persona context the normal chat path uses,
15
+ * a natural seam for the V2 realtime-provider adapter (swap this
16
+ function, everything else stays).
17
+
18
+ When the feature flag is off this module is never imported.
19
+ """
20
+ from __future__ import annotations
21
+
22
+ import os
23
+ from typing import Any, Dict, List, Optional
24
+
25
+ import httpx
26
+
27
+
28
+ def _local_backend_url() -> str:
29
+ """Where to reach our own FastAPI server over loopback. Honors an
30
+ override for containerized / split-process deployments."""
31
+ return (
32
+ os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL")
33
+ or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL")
34
+ or "http://127.0.0.1:8000"
35
+ ).rstrip("/")
36
+
37
+
38
+ DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "30"))
39
+
40
+
41
+ async def run_turn(
42
+ *,
43
+ user_text: str,
44
+ model: str,
45
+ auth_bearer: Optional[str] = None,
46
+ system_prompt: Optional[str] = None,
47
+ additional_system: Optional[str] = None,
48
+ history: Optional[List[Dict[str, str]]] = None,
49
+ extra_headers: Optional[Dict[str, str]] = None,
50
+ ) -> str:
51
+ """Run a single assistant turn via the existing chat endpoint.
52
+
53
+ Parameters
54
+ ----------
55
+ user_text:
56
+ The final STT transcript from the client.
57
+ model:
58
+ The model id. ``persona:<project_id>`` / ``personality:<id>`` /
59
+ plain ollama model name β€” whatever the existing chat endpoint
60
+ accepts.
61
+ auth_bearer:
62
+ The caller's bearer token, forwarded verbatim so the chat
63
+ endpoint sees the same user identity as the WS session owner.
64
+ system_prompt / history:
65
+ Optional pre-built context. Left empty the endpoint will attach
66
+ its own persona / memory context based on the model.
67
+ additional_system:
68
+ Optional extra system message appended AFTER ``system_prompt``
69
+ (or on its own if no ``system_prompt`` is provided). Additive
70
+ only β€” used by ``persona_call`` to inject a per-turn phone-call
71
+ suffix WITHOUT touching the persona's own system prompt that
72
+ the compat endpoint already injects for ``persona:*`` models.
73
+
74
+ Returns
75
+ -------
76
+ The assistant's reply text (the ``choices[0].message.content`` from
77
+ the chat completion response). Raises on non-2xx responses so the
78
+ caller can surface an ``error`` event over the WS.
79
+ """
80
+ messages: List[Dict[str, str]] = []
81
+ if system_prompt:
82
+ messages.append({"role": "system", "content": system_prompt})
83
+ if additional_system:
84
+ messages.append({"role": "system", "content": additional_system})
85
+ if history:
86
+ messages.extend(history)
87
+ messages.append({"role": "user", "content": user_text})
88
+
89
+ headers: Dict[str, str] = {"Content-Type": "application/json"}
90
+ if auth_bearer:
91
+ headers["Authorization"] = f"Bearer {auth_bearer}"
92
+ if extra_headers:
93
+ headers.update(extra_headers)
94
+
95
+ payload = {
96
+ "model": model,
97
+ "messages": messages,
98
+ # Final-only β€” must stay False at MVP. The compat endpoint
99
+ # returns 501 on streaming requests.
100
+ "stream": False,
101
+ # Keep response tight for voice. Long prose makes for bad audio.
102
+ "max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")),
103
+ "temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")),
104
+ }
105
+
106
+ url = f"{_local_backend_url()}/v1/chat/completions"
107
+ async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client:
108
+ r = await client.post(url, json=payload, headers=headers)
109
+ # Let the caller see the HTTP body on error β€” useful for debugging.
110
+ if r.status_code >= 400:
111
+ raise RuntimeError(
112
+ f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}"
113
+ )
114
+ data = r.json()
115
+ try:
116
+ return (
117
+ data["choices"][0]["message"]["content"]
118
+ if data.get("choices")
119
+ else ""
120
+ )
121
+ except (KeyError, IndexError, TypeError):
122
+ raise RuntimeError(
123
+ "chat endpoint returned an unexpected body shape: "
124
+ + str(data)[:400]
125
+ )
backend/app/voice_call/turn_stream.py ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ Streaming turn runner β€” async-generator peer of :func:`turn.run_turn`.
3
+
4
+ Yields assistant text chunks from the chat provider as they arrive so
5
+ the WebSocket handler can emit ``assistant.partial`` envelopes without
6
+ waiting for the full reply. Drops cleanly on barge-in cancel.
7
+
8
+ Contract is documented in
9
+ ``docs/analysis/voice-call-streaming-design.md`` Β§ 4.1 and Β§ 4.2.
10
+
11
+ Two execution paths, selected at runtime:
12
+
13
+ 1. **Native streaming** β€” POST to the compat chat endpoint with
14
+ ``stream: true`` and consume the SSE / NDJSON response. This is
15
+ the real Phase 2 path and delivers sub-500 ms first-audio when
16
+ the provider supports it.
17
+
18
+ 2. **Chunked-unary fallback** β€” if the compat endpoint returns 501
19
+ (current default, see ``turn.py`` comment) OR the provider
20
+ doesn't support streaming, we call the unary endpoint, receive
21
+ the full reply, and emit it in clause-sized chunks with a tiny
22
+ delay so the downstream plumbing (``assistant.partial`` β†’
23
+ ``streamTts.appendDelta``) still exercises correctly.
24
+
25
+ The fallback yields no latency improvement β€” its value is keeping
26
+ the envelope contract and the client-side streaming TTS alive even
27
+ while the compat endpoint hasn't flipped on streaming. When it does,
28
+ this module starts getting real deltas automatically.
29
+
30
+ The persona_call suffix, persona resolution, and auth are all handled
31
+ by ``ws.py`` / the compat endpoint exactly as today. This module
32
+ receives the already-resolved ``user_text`` + ``model`` + headers
33
+ and does nothing with the persona layer directly.
34
+ """
35
+ from __future__ import annotations
36
+
37
+ import asyncio
38
+ import json
39
+ import os
40
+ import re
41
+ from typing import AsyncIterator, Dict, List, Optional
42
+
43
+ import httpx
44
+
45
+ from .barge_in import BargeInToken
46
+
47
+
48
+ def _local_backend_url() -> str:
49
+ return (
50
+ os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL")
51
+ or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL")
52
+ or "http://127.0.0.1:8000"
53
+ ).rstrip("/")
54
+
55
+
56
+ DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "60"))
57
+ # Chunk pacing for the fallback path. Kept deliberately small so the
58
+ # fallback "feels like" streaming without flooding the client.
59
+ _FALLBACK_INTER_CHUNK_MS = int(os.getenv("VOICE_CALL_FALLBACK_CHUNK_MS", "35"))
60
+
61
+ # Simple clause splitter used only by the fallback path. Deliberately
62
+ # naive: split on sentence-enders first, then on commas if a sentence
63
+ # is too long for one chunk.
64
+ _SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+")
65
+ _CHUNK_MAX_CHARS = 90
66
+
67
+
68
+ def _split_for_chunking(text: str) -> List[str]:
69
+ if not text:
70
+ return []
71
+ parts = [p for p in _SENTENCE_SPLIT.split(text) if p]
72
+ out: List[str] = []
73
+ for part in parts:
74
+ if len(part) <= _CHUNK_MAX_CHARS:
75
+ out.append(part)
76
+ continue
77
+ # Long sentence β€” split on commas, rejoining short pieces so
78
+ # we don't over-chunk.
79
+ comma_bits = [b.strip() for b in part.split(",") if b.strip()]
80
+ buf = ""
81
+ for bit in comma_bits:
82
+ if len(buf) + len(bit) + 2 <= _CHUNK_MAX_CHARS:
83
+ buf = f"{buf}, {bit}" if buf else bit
84
+ else:
85
+ if buf:
86
+ out.append(buf)
87
+ buf = bit
88
+ if buf:
89
+ out.append(buf)
90
+ # Re-attach trailing whitespace so concatenation preserves spacing.
91
+ return [f"{p} " if not p.endswith(" ") else p for p in out]
92
+
93
+
94
+ def _extract_delta_from_sse_line(line: str) -> Optional[str]:
95
+ """Parse one SSE / NDJSON frame from a streaming chat endpoint.
96
+
97
+ Supports both the OpenAI-style ``data: {...}`` SSE format and
98
+ Ollama's raw NDJSON (``{"message": {...}}`` per line).
99
+ """
100
+ line = line.strip()
101
+ if not line:
102
+ return None
103
+ if line.startswith("data:"):
104
+ line = line[5:].strip()
105
+ if line == "[DONE]":
106
+ return None
107
+ try:
108
+ obj = json.loads(line)
109
+ except json.JSONDecodeError:
110
+ return None
111
+ # OpenAI streaming: choices[0].delta.content
112
+ try:
113
+ choices = obj.get("choices") or []
114
+ if choices:
115
+ delta = (choices[0] or {}).get("delta") or {}
116
+ content = delta.get("content")
117
+ if isinstance(content, str) and content:
118
+ return content
119
+ except (AttributeError, TypeError):
120
+ pass
121
+ # Ollama streaming: message.content
122
+ try:
123
+ msg = obj.get("message") or {}
124
+ content = msg.get("content")
125
+ if isinstance(content, str) and content:
126
+ return content
127
+ except (AttributeError, TypeError):
128
+ pass
129
+ return None
130
+
131
+
132
+ async def _yield_native_stream(
133
+ client: httpx.AsyncClient,
134
+ url: str,
135
+ payload: Dict,
136
+ headers: Dict[str, str],
137
+ cancel_token: BargeInToken,
138
+ ) -> AsyncIterator[str]:
139
+ """Native provider streaming path β€” returns an async iterator of
140
+ deltas. Caller handles 501 / 4xx / 5xx by catching the upstream
141
+ status before entering this coroutine."""
142
+ async with client.stream("POST", url, json=payload, headers=headers) as r:
143
+ if r.status_code == 501:
144
+ # Sentinel for "provider doesn't support streaming yet".
145
+ # Caller will fall back to chunked-unary.
146
+ await r.aread()
147
+ raise _StreamingUnsupported()
148
+ if r.status_code >= 400:
149
+ body = (await r.aread()).decode("utf-8", errors="replace")
150
+ raise RuntimeError(
151
+ f"chat stream {url} returned {r.status_code}: {body[:400]}"
152
+ )
153
+ async for line in r.aiter_lines():
154
+ if cancel_token.is_cancelled():
155
+ return
156
+ delta = _extract_delta_from_sse_line(line)
157
+ if delta:
158
+ yield delta
159
+
160
+
161
+ class _StreamingUnsupported(Exception):
162
+ """Raised when the provider returns 501 on ``stream: true`` so the
163
+ caller can route to the chunked-unary fallback without conflating
164
+ it with an ordinary upstream error."""
165
+
166
+
167
+ async def _yield_chunked_unary(
168
+ client: httpx.AsyncClient,
169
+ url: str,
170
+ payload: Dict,
171
+ headers: Dict[str, str],
172
+ cancel_token: BargeInToken,
173
+ ) -> AsyncIterator[str]:
174
+ """Fallback path β€” full unary call, then chunk the reply."""
175
+ unary_payload = dict(payload, stream=False)
176
+ r = await client.post(url, json=unary_payload, headers=headers)
177
+ if r.status_code >= 400:
178
+ raise RuntimeError(
179
+ f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}"
180
+ )
181
+ data = r.json()
182
+ try:
183
+ reply = (
184
+ data["choices"][0]["message"]["content"]
185
+ if data.get("choices")
186
+ else ""
187
+ )
188
+ except (KeyError, IndexError, TypeError):
189
+ raise RuntimeError(
190
+ f"chat endpoint returned an unexpected body shape: {str(data)[:400]}"
191
+ )
192
+ for chunk in _split_for_chunking(reply):
193
+ if cancel_token.is_cancelled():
194
+ return
195
+ yield chunk
196
+ # Small delay so the client's streaming TTS has a beat to
197
+ # synth + play before the next chunk arrives. Keeps the
198
+ # fallback from flooding the WS.
199
+ await asyncio.sleep(_FALLBACK_INTER_CHUNK_MS / 1000.0)
200
+
201
+
202
+ async def run_turn_streaming(
203
+ *,
204
+ user_text: str,
205
+ model: str,
206
+ auth_bearer: Optional[str] = None,
207
+ system_prompt: Optional[str] = None,
208
+ additional_system: Optional[str] = None,
209
+ history: Optional[List[Dict[str, str]]] = None,
210
+ extra_headers: Optional[Dict[str, str]] = None,
211
+ cancel_token: BargeInToken,
212
+ ) -> AsyncIterator[str]:
213
+ """Yield assistant text chunks for the given user turn.
214
+
215
+ Exits cleanly (no exception) when ``cancel_token`` is tripped by
216
+ a barge-in. The caller is responsible for emitting the final
217
+ ``assistant.turn_end`` envelope regardless of exit reason.
218
+ """
219
+ messages: List[Dict[str, str]] = []
220
+ if system_prompt:
221
+ messages.append({"role": "system", "content": system_prompt})
222
+ if additional_system:
223
+ messages.append({"role": "system", "content": additional_system})
224
+ if history:
225
+ messages.extend(history)
226
+ messages.append({"role": "user", "content": user_text})
227
+
228
+ headers: Dict[str, str] = {"Content-Type": "application/json"}
229
+ if auth_bearer:
230
+ headers["Authorization"] = f"Bearer {auth_bearer}"
231
+ if extra_headers:
232
+ headers.update(extra_headers)
233
+
234
+ payload = {
235
+ "model": model,
236
+ "messages": messages,
237
+ "stream": True,
238
+ "max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")),
239
+ "temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")),
240
+ }
241
+
242
+ url = f"{_local_backend_url()}/v1/chat/completions"
243
+
244
+ async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client:
245
+ # Try native streaming first.
246
+ try:
247
+ async for delta in _yield_native_stream(
248
+ client, url, payload, headers, cancel_token,
249
+ ):
250
+ yield delta
251
+ return
252
+ except _StreamingUnsupported:
253
+ # Compat endpoint currently returns 501 on stream=true;
254
+ # fall back to unary + chunker. Covered by test.
255
+ pass
256
+ async for delta in _yield_chunked_unary(
257
+ client, url, payload, headers, cancel_token,
258
+ ):
259
+ yield delta
backend/app/voice_call/ws.py ADDED
@@ -0,0 +1,605 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ WebSocket orchestrator for voice-call sessions.
3
+
4
+ Transport
5
+ ---------
6
+ Path: ``/v1/voice-call/ws/{session_id}``
7
+ Query param: ``resume_token=<token issued at POST /sessions>``
8
+ (headers don't round-trip cleanly from browsers; a
9
+ short-lived query token is the standard WS-auth trick)
10
+
11
+ Protocol
12
+ --------
13
+ Server and client exchange JSON envelopes::
14
+
15
+ { "type": "<event>", "seq": <int?>, "ts": <ms>, "payload": { ... } }
16
+
17
+ Client β†’ server (each is harmless if the server can't handle it β€”
18
+ forward-compatible by design):
19
+ ui.state { muted, speaker_on, backgrounded }
20
+ transcript.final { text, lang? }
21
+ call.control { action: "end" | "ping" }
22
+
23
+ Server β†’ client:
24
+ call.state { status, since }
25
+ transcript.final { role: "assistant", text }
26
+ error { code, message }
27
+ pong { ts }
28
+ safety.notice (forwarded from upstream providers, reserved)
29
+
30
+ Out of scope for MVP (per review): transcript.partial (requires chat
31
+ endpoint streaming, which returns 501 today), raw audio frames, server-
32
+ side barge-in.
33
+ """
34
+ from __future__ import annotations
35
+
36
+ import asyncio
37
+ import json
38
+ import logging
39
+ import time
40
+ import uuid
41
+ from typing import Any, Dict, Optional
42
+
43
+ from fastapi import WebSocket, WebSocketDisconnect, status
44
+
45
+ from . import barge_in as _barge_in
46
+ from . import service, store, turn, turn_stream
47
+ from .config import VoiceCallConfig
48
+
49
+
50
+ logger = logging.getLogger("voice_call.ws")
51
+
52
+
53
+ def _clog(event: str, **kwargs: Any) -> None:
54
+ """Structured ``[call]`` log line β€” mirrors the frontend's clog()
55
+ format so a single grep surfaces the whole round-trip. One
56
+ ``logger.info`` per inflection point; fields stay flat so
57
+ downstream log-shippers don't have to deserialize.
58
+
59
+ Used at: session accept (open), call.state live (ready), opener
60
+ greeting (greet), user transcript in (turn_user_in), assistant
61
+ reply out (turn_assistant_out), cooperative cancel (barge_in),
62
+ user-ended (hangup), WS teardown (close)."""
63
+ logger.info("[call] %s %s", event, json.dumps(kwargs, separators=(",", ":"), default=str))
64
+
65
+ HEARTBEAT_INTERVAL_SEC = 10.0
66
+
67
+ # Protocol sentinels β€” reserved strings the frontend may send as a
68
+ # ``transcript.final`` payload to nudge backend state without asking
69
+ # the LLM to react. We drop these on the floor before the turn runner
70
+ # sees them; otherwise the model treats them as literal user speech
71
+ # and produces a confused reply (the classic "I'm not sure what
72
+ # [phone-call-open] means" double-greet). Add new sentinels here only
73
+ # when both sides have landed.
74
+ _PROTOCOL_SENTINELS = frozenset({
75
+ "[phone-call-open]",
76
+ "[phone-call-close]",
77
+ })
78
+
79
+
80
+ def _now_ms() -> int:
81
+ return int(time.time() * 1000)
82
+
83
+
84
+ async def _send(ws: WebSocket, type_: str, payload: Dict[str, Any], *,
85
+ session_id: str) -> None:
86
+ """Send a server→client event with a monotonic seq number."""
87
+ seq = service._next_event_seq(session_id)
88
+ env = {"type": type_, "seq": seq, "ts": _now_ms(), "payload": payload}
89
+ try:
90
+ await ws.send_text(json.dumps(env, separators=(",", ":")))
91
+ except Exception:
92
+ # WebSocket already closed; caller will clean up.
93
+ raise
94
+ # Only persist the *type* for privacy unless the user has explicitly
95
+ # opted into transcript storage.
96
+ store.append_event(
97
+ session_id=session_id, seq=seq, event_type=type_,
98
+ payload={}, # intentionally empty
99
+ )
100
+
101
+
102
+ async def _send_error(ws: WebSocket, session_id: str, code: str, msg: str) -> None:
103
+ try:
104
+ await _send(ws, "error",
105
+ {"code": code, "message": msg},
106
+ session_id=session_id)
107
+ except Exception:
108
+ pass # best-effort
109
+
110
+
111
+ async def _resolve_session_for_ws(
112
+ ws: WebSocket, session_id: str, resume_token: Optional[str],
113
+ cfg: VoiceCallConfig,
114
+ ) -> Optional[Dict[str, Any]]:
115
+ """Validate the handshake. Closes the socket with a clean code if
116
+ anything is off."""
117
+ if not cfg.websocket_enabled:
118
+ await ws.close(code=status.WS_1008_POLICY_VIOLATION,
119
+ reason="websocket-disabled")
120
+ return None
121
+ row = store._select_session(session_id)
122
+ if row is None:
123
+ await ws.close(code=status.WS_1008_POLICY_VIOLATION,
124
+ reason="session-not-found")
125
+ return None
126
+ if not resume_token or row.get("resume_token") != resume_token:
127
+ await ws.close(code=status.WS_1008_POLICY_VIOLATION,
128
+ reason="bad-resume-token")
129
+ return None
130
+ if row.get("status") == "ended":
131
+ await ws.close(code=status.WS_1008_POLICY_VIOLATION,
132
+ reason="session-ended")
133
+ return None
134
+ # Resume window: an "interrupted" session is only acceptable if the
135
+ # drop happened within the configured window.
136
+ if row.get("status") == "interrupted":
137
+ last_update = int(row.get("updated_at") or 0)
138
+ if _now_ms() - last_update > cfg.resume_window_sec * 1000:
139
+ service.mark_ended_by_timeout(session_id, "resume_expired")
140
+ await ws.close(code=status.WS_1008_POLICY_VIOLATION,
141
+ reason="resume-expired")
142
+ return None
143
+ return row
144
+
145
+
146
+ async def run_session_ws(
147
+ ws: WebSocket,
148
+ session_id: str,
149
+ resume_token: Optional[str],
150
+ cfg: VoiceCallConfig,
151
+ ) -> None:
152
+ """Main WS loop for one session. Called from router after
153
+ ``await ws.accept()``.
154
+
155
+ Ownership is established by the resume token alone (token is only
156
+ ever handed to the user who created the session). The per-turn
157
+ chat request also carries the caller's bearer β€” so the chat
158
+ endpoint does its own auth and refuses foreign personas even if
159
+ the token somehow leaks."""
160
+ session = await _resolve_session_for_ws(ws, session_id, resume_token, cfg)
161
+ if session is None:
162
+ return # close already issued
163
+
164
+ # Extract the bearer from the WS query so turn.py can re-use it.
165
+ # Browsers can't attach custom headers to WS; the client passes
166
+ # ?token=<jwt> alongside ?resume_token=.
167
+ query_bearer = ws.query_params.get("token") or None
168
+
169
+ sid = session["id"]
170
+ _clog("lifecycle_open", sid=sid, persona_id=session.get("persona_id"))
171
+ service.mark_live(sid)
172
+ await _send(ws, "call.state", {"status": "live"}, session_id=sid)
173
+ _clog("lifecycle_ready", sid=sid)
174
+
175
+ # ── "Theory of answering" hook ────────────────────────────────
176
+ # When persona_call is on, the persona answers first β€” the way a
177
+ # real callee picks up the phone β€” instead of sitting silent
178
+ # until the caller speaks. The opening text comes from the
179
+ # curated bank in persona_call/openings.py; selection is
180
+ # deterministic (seeded by session_id) + persona-aware + rotated
181
+ # against the anti-repetition ledger, so the same caller never
182
+ # hears the same greeting twice in a row.
183
+ try:
184
+ from ..persona_call import (
185
+ config as _pc_cfg_mod,
186
+ context as _pc_context,
187
+ facets as _pc_facets,
188
+ openings as _pc_openings,
189
+ store as _pc_store,
190
+ )
191
+ _pc_cfg = _pc_cfg_mod.load()
192
+ if _pc_cfg.enabled:
193
+ _pc_store.ensure_schema()
194
+ _pc_store.ensure_state(sid)
195
+ _state = _pc_store.get_state(sid) or {}
196
+ forbidden = list(_state.get("recent_openers") or [])
197
+ _pc_env = _pc_context.compute_env(tz=None, weeks_since_last_call=-1)
198
+ _pc_facets_obj = _pc_facets.for_persona_id(session.get("persona_id"))
199
+ # We need the template *id* too so we can push it into
200
+ # the ledger, so call choose() (not render() directly).
201
+ tpl = _pc_openings.choose(
202
+ _pc_facets_obj, _pc_env,
203
+ session_id=sid,
204
+ turn_index=1,
205
+ forbidden_ids=forbidden,
206
+ )
207
+ greeting_text = None
208
+ if tpl is not None:
209
+ greeting_text = _pc_openings._interpolate(
210
+ tpl.text,
211
+ # Fall back to "Assistant" only if the session row
212
+ # has no persona_id β€” matches the frontend label.
213
+ (session.get("persona_id") or "Assistant").replace("persona:", "").strip(),
214
+ )
215
+ # Append the chosen id to the opener ledger, keeping
216
+ # only the last N so the ledger doesn't grow
217
+ # unbounded across long-lived sessions.
218
+ window = max(1, _pc_cfg.opener_ledger_window)
219
+ new_openers = (list(forbidden) + [tpl.id])[-window:]
220
+ _pc_store.update_state(sid, recent_openers=new_openers)
221
+ if greeting_text:
222
+ await _send(ws, "transcript.final",
223
+ {"role": "assistant", "text": greeting_text},
224
+ session_id=sid)
225
+ _clog("lifecycle_greet", sid=sid, text_len=len(greeting_text))
226
+ except Exception as _pc_err:
227
+ # Shadow-fail β€” the call still works without a greeting.
228
+ logger.warning("[persona_call] opening greeting skipped: %s", _pc_err)
229
+
230
+ # Heartbeat keeps idle TCP paths alive on mobile networks.
231
+ hb_task = asyncio.create_task(_heartbeat_loop(ws, sid, cfg))
232
+
233
+ try:
234
+ while True:
235
+ try:
236
+ text = await asyncio.wait_for(
237
+ ws.receive_text(),
238
+ timeout=cfg.idle_timeout_sec,
239
+ )
240
+ except asyncio.TimeoutError:
241
+ await _send_error(ws, sid, "idle_timeout",
242
+ "No activity for a while β€” ending call.")
243
+ service.end_session(sid=sid, user_id=session["user_id"],
244
+ reason="idle")
245
+ await _send(ws, "call.state", {"status": "ended",
246
+ "reason": "idle"},
247
+ session_id=sid)
248
+ break
249
+
250
+ try:
251
+ env = json.loads(text)
252
+ except json.JSONDecodeError:
253
+ await _send_error(ws, sid, "bad_json", "Invalid JSON envelope.")
254
+ continue
255
+
256
+ evt_type = env.get("type")
257
+ payload = env.get("payload") or {}
258
+
259
+ if evt_type == "call.control":
260
+ action = (payload.get("action") or "").strip()
261
+ if action == "end":
262
+ service.end_session(sid=sid, user_id=session["user_id"],
263
+ reason="user_ended")
264
+ await _send(ws, "call.state", {"status": "ended",
265
+ "reason": "user_ended"},
266
+ session_id=sid)
267
+ _clog("lifecycle_hangup", sid=sid, reason="user_ended")
268
+ break
269
+ if action == "ping":
270
+ await _send(ws, "pong", {}, session_id=sid)
271
+
272
+ elif evt_type == "ui.state":
273
+ # Pure bookkeeping. Client tells us muted / speaker_on /
274
+ # backgrounded; we persist only the type (privacy).
275
+ pass
276
+
277
+ elif evt_type == "transcript.final":
278
+ text_in = (payload.get("text") or "").strip()
279
+ if not text_in:
280
+ continue
281
+ # Protocol sentinels are NOT user utterances β€” they
282
+ # exist solely so the frontend can signal state
283
+ # transitions (phase machine nudges, etc.) without
284
+ # the LLM reacting to the literal string. Drop and
285
+ # continue; the phase machine already advances on the
286
+ # first real user utterance, so no special handling
287
+ # is required here beyond not running the turn.
288
+ if text_in in _PROTOCOL_SENTINELS:
289
+ continue
290
+ _clog("turn_user_in", sid=sid, text_len=len(text_in))
291
+ model = (
292
+ payload.get("model")
293
+ or session.get("persona_id")
294
+ and f"persona:{session['persona_id']}"
295
+ or "llama3:8b"
296
+ )
297
+
298
+ # ── persona_call hook ───────────────────────────────
299
+ # Additive, behind its own feature flag. Composes a
300
+ # per-turn system SUFFIX (never touches the persona's
301
+ # base system prompt) and β€” if cfg.apply is True β€”
302
+ # appends it as an extra system message on the chat
303
+ # call. Wraps the turn in a filler scheduler so a long
304
+ # LLM wait produces a "hmm…" event within 600 ms.
305
+ pc_additional_system: Optional[str] = None
306
+ pc_persona_id: Optional[str] = session.get("persona_id")
307
+ try:
308
+ from ..persona_call import (
309
+ config as _pc_cfg_mod,
310
+ context as _pc_context,
311
+ directive as _pc_directive,
312
+ facets as _pc_facets,
313
+ latency as _pc_latency,
314
+ )
315
+ _pc_cfg = _pc_cfg_mod.load()
316
+ except Exception:
317
+ _pc_cfg = None # persona_call not importable; skip
318
+
319
+ _pc_facets_obj = None
320
+ if _pc_cfg is not None and _pc_cfg.enabled:
321
+ try:
322
+ _pc_env = _pc_context.compute_env(
323
+ tz=(session.get("client_platform") and None),
324
+ weeks_since_last_call=-1,
325
+ )
326
+ _pc_facets_obj = _pc_facets.for_persona_id(pc_persona_id)
327
+ composed = _pc_directive.compose(
328
+ session_id=sid,
329
+ persona_id=pc_persona_id,
330
+ user_text=text_in,
331
+ env=_pc_env,
332
+ cfg=_pc_cfg,
333
+ facets=_pc_facets_obj,
334
+ )
335
+ if _pc_cfg.apply and composed.system_suffix:
336
+ pc_additional_system = composed.system_suffix
337
+ except Exception as _pc_err:
338
+ # Shadow fail β€” turn still runs without the
339
+ # phone-call suffix. persona_call must never
340
+ # break an otherwise-working call.
341
+ logger.warning(
342
+ "[persona_call] compose skipped: %s", _pc_err
343
+ )
344
+
345
+ async def _pc_send(env_: dict) -> None:
346
+ try:
347
+ await ws.send_text(
348
+ json.dumps(env_, separators=(",", ":"))
349
+ )
350
+ except Exception:
351
+ pass
352
+
353
+ # ── Streaming vs unary turn β€” additive branch ──────
354
+ # When VOICE_CALL_STREAMING_ENABLED=true the turn runs
355
+ # through turn_stream.run_turn_streaming and emits
356
+ # ``assistant.partial`` + ``assistant.turn_end`` envelopes
357
+ # so the client's streaming TTS can begin speaking long
358
+ # before the full reply has arrived. When the flag is
359
+ # off we take the original path byte-for-byte.
360
+ if cfg.streaming_enabled:
361
+ turn_id = f"t_{uuid.uuid4().hex[:12]}"
362
+ token = _barge_in.new_token(sid, turn_id)
363
+ assistant_text_parts: list[str] = []
364
+ end_reason = "complete"
365
+ try:
366
+ if (
367
+ _pc_cfg is not None
368
+ and _pc_cfg.enabled
369
+ and _pc_facets_obj is not None
370
+ ):
371
+ # Filler scheduler still wraps streaming β€”
372
+ # if the FIRST delta takes > N ms the
373
+ # client still gets a ``hmm…`` event.
374
+ async with _pc_latency.FillerScheduler(
375
+ send=_pc_send,
376
+ facets=_pc_facets_obj,
377
+ cfg=_pc_cfg,
378
+ session_id=sid,
379
+ ):
380
+ index = 0
381
+ async for delta in turn_stream.run_turn_streaming(
382
+ user_text=text_in,
383
+ model=model,
384
+ auth_bearer=query_bearer,
385
+ additional_system=pc_additional_system,
386
+ cancel_token=token,
387
+ ):
388
+ if token.is_cancelled():
389
+ break
390
+ assistant_text_parts.append(delta)
391
+ await _send(
392
+ ws, "assistant.partial",
393
+ {
394
+ "turn_id": turn_id,
395
+ "delta": delta,
396
+ "index": index,
397
+ },
398
+ session_id=sid,
399
+ )
400
+ index += 1
401
+ else:
402
+ index = 0
403
+ async for delta in turn_stream.run_turn_streaming(
404
+ user_text=text_in,
405
+ model=model,
406
+ auth_bearer=query_bearer,
407
+ additional_system=pc_additional_system,
408
+ cancel_token=token,
409
+ ):
410
+ if token.is_cancelled():
411
+ break
412
+ assistant_text_parts.append(delta)
413
+ await _send(
414
+ ws, "assistant.partial",
415
+ {
416
+ "turn_id": turn_id,
417
+ "delta": delta,
418
+ "index": index,
419
+ },
420
+ session_id=sid,
421
+ )
422
+ index += 1
423
+ if token.is_cancelled():
424
+ end_reason = "cancelled"
425
+ except Exception as exc:
426
+ logger.warning("[voice_call] stream turn failed: %s", exc)
427
+ end_reason = "error"
428
+ finally:
429
+ # Always emit turn_end so the client can close
430
+ # the turn bookkeeping cleanly.
431
+ _barge_in.clear_session(sid)
432
+ assistant_text = "".join(assistant_text_parts)
433
+ await _send(
434
+ ws, "assistant.turn_end",
435
+ {
436
+ "turn_id": turn_id,
437
+ "reason": end_reason,
438
+ "full_text": assistant_text,
439
+ },
440
+ session_id=sid,
441
+ )
442
+ _clog(
443
+ "turn_assistant_out",
444
+ sid=sid,
445
+ turn_id=turn_id,
446
+ reason=end_reason,
447
+ text_len=len(assistant_text),
448
+ route="stream",
449
+ )
450
+ else:
451
+ try:
452
+ if (
453
+ _pc_cfg is not None
454
+ and _pc_cfg.enabled
455
+ and _pc_facets_obj is not None
456
+ ):
457
+ async with _pc_latency.FillerScheduler(
458
+ send=_pc_send,
459
+ facets=_pc_facets_obj,
460
+ cfg=_pc_cfg,
461
+ session_id=sid,
462
+ ):
463
+ assistant_text = await turn.run_turn(
464
+ user_text=text_in,
465
+ model=model,
466
+ auth_bearer=query_bearer,
467
+ additional_system=pc_additional_system,
468
+ )
469
+ else:
470
+ assistant_text = await turn.run_turn(
471
+ user_text=text_in,
472
+ model=model,
473
+ auth_bearer=query_bearer,
474
+ additional_system=pc_additional_system,
475
+ )
476
+ except Exception as exc:
477
+ logger.warning("[voice_call] turn failed: %s", exc)
478
+ await _send_error(ws, sid, "turn_failed", str(exc)[:400])
479
+ continue
480
+
481
+ await _send(ws, "transcript.final",
482
+ {"role": "assistant", "text": assistant_text},
483
+ session_id=sid)
484
+ _clog(
485
+ "turn_assistant_out",
486
+ sid=sid,
487
+ text_len=len(assistant_text),
488
+ route="unary",
489
+ )
490
+
491
+ # Record the reply into the anti-repetition ledger so
492
+ # the NEXT turn can forbid the just-used opener/ack.
493
+ if _pc_cfg is not None and _pc_cfg.enabled:
494
+ try:
495
+ _pc_directive.record_persona_reply(
496
+ session_id=sid,
497
+ persona_id=pc_persona_id,
498
+ reply=assistant_text,
499
+ cfg=_pc_cfg,
500
+ )
501
+ except Exception as _pc_err:
502
+ logger.warning(
503
+ "[persona_call] record_reply skipped: %s",
504
+ _pc_err,
505
+ )
506
+
507
+ elif evt_type == "user.barge_in":
508
+ # Phase 3 β€” cooperative cancel. Client's VAD detected
509
+ # user speech-start while the persona was mid-reply.
510
+ # Β§ 4.4 of the streaming design doc.
511
+ if not (cfg.streaming_enabled and cfg.barge_in_enabled):
512
+ continue
513
+ turn_id = (payload.get("turn_id") or "").strip()
514
+ if not turn_id:
515
+ continue
516
+ cancelled = _barge_in.cancel_active(sid, turn_id)
517
+ if cancelled:
518
+ # Emit the ack IMMEDIATELY β€” client stops TTS on
519
+ # receipt, shaving ~50-100 ms off the perceived
520
+ # barge-in latency vs waiting for the generator
521
+ # to notice the event on its next poll.
522
+ await _send(
523
+ ws, "assistant.cancel",
524
+ {"turn_id": turn_id, "cause": "user_barge_in"},
525
+ session_id=sid,
526
+ )
527
+ _clog("barge_in", sid=sid, turn_id=turn_id, cause="user_barge_in")
528
+ # Else: stale turn_id (turn already ended). Silent drop.
529
+
530
+ elif evt_type == "transcript.partial":
531
+ # Secondary barge-in signal β€” if the client's explicit
532
+ # user.barge_in didn't arrive but the STT started
533
+ # decoding words, that also means the user spoke.
534
+ # Only fires while a turn is active.
535
+ if not (cfg.streaming_enabled and cfg.barge_in_enabled):
536
+ continue
537
+ active = _barge_in.get_active(sid)
538
+ if active is not None and not active.is_cancelled():
539
+ cancelled = _barge_in.cancel_active(sid, active.turn_id)
540
+ if cancelled:
541
+ await _send(
542
+ ws, "assistant.cancel",
543
+ {
544
+ "turn_id": active.turn_id,
545
+ "cause": "user_partial",
546
+ },
547
+ session_id=sid,
548
+ )
549
+
550
+ else:
551
+ # Forward-compat: unknown event type is a no-op, not an
552
+ # error. Older servers + newer clients stay friendly.
553
+ pass
554
+ except WebSocketDisconnect:
555
+ # Client dropped β€” park the session as 'interrupted' so a resume
556
+ # request within the window can pick it back up.
557
+ _barge_in.clear_session(sid)
558
+ service.mark_interrupted(sid)
559
+ except Exception as exc:
560
+ logger.exception("[voice_call] ws error for %s: %s", sid, exc)
561
+ try:
562
+ await _send_error(ws, sid, "internal", "Internal error")
563
+ except Exception:
564
+ pass
565
+ _barge_in.clear_session(sid)
566
+ service.mark_interrupted(sid)
567
+ finally:
568
+ _clog("lifecycle_close", sid=sid)
569
+ _barge_in.clear_session(sid)
570
+ hb_task.cancel()
571
+ try:
572
+ await hb_task
573
+ except Exception:
574
+ pass
575
+
576
+
577
+ async def _heartbeat_loop(ws: WebSocket, session_id: str, cfg: VoiceCallConfig) -> None:
578
+ """Send a server-initiated ping every N seconds. Doubles as a
579
+ liveness check + hard-duration-cap enforcement."""
580
+ while True:
581
+ await asyncio.sleep(HEARTBEAT_INTERVAL_SEC)
582
+ row = store._select_session(session_id)
583
+ if row is None or row["status"] == "ended":
584
+ return
585
+ # Enforce hard cap.
586
+ started = int(row.get("started_at") or 0)
587
+ if started and (_now_ms() - started) >= cfg.max_duration_sec * 1000:
588
+ try:
589
+ await _send(ws, "call.state",
590
+ {"status": "ending", "reason": "max_duration"},
591
+ session_id=session_id)
592
+ except Exception:
593
+ return
594
+ service.end_session(sid=session_id, user_id=row["user_id"],
595
+ reason="max_duration")
596
+ try:
597
+ await ws.close(code=status.WS_1000_NORMAL_CLOSURE,
598
+ reason="max-duration")
599
+ except Exception:
600
+ pass
601
+ return
602
+ try:
603
+ await _send(ws, "ping", {}, session_id=session_id)
604
+ except Exception:
605
+ return
frontend/.cache-bust CHANGED
@@ -1 +1 @@
1
- 1776444513
 
1
+ 1776516992
frontend/src/ui/App.tsx CHANGED
@@ -21,6 +21,7 @@ import {
21
  Copy,
22
  RotateCw,
23
  PenLine,
 
24
  Users,
25
  EyeOff,
26
  BookOpen,
@@ -34,6 +35,9 @@ import UserAvatar from './components/UserAvatar'
34
  import AccountMenu, { type AccountMenuUser } from './components/AccountMenu'
35
  import { useAuth } from './components/AuthGate'
36
  import VoiceMode, { stripMarkdownForSpeech } from './VoiceModeGrok'
 
 
 
37
  // Legacy voice mode available as: import VoiceModeLegacy from './VoiceModeLegacy'
38
  import ProjectsView from './ProjectsView'
39
  import ImagineView from './Imagine'
@@ -113,6 +117,22 @@ export type Msg = {
113
  intent: 'generate_images' | 'generate_videos'
114
  prompt: string
115
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
116
  }
117
 
118
  // ---------------------------------------------------------------------------
@@ -1574,12 +1594,92 @@ function useCopyMessage(timeoutMs = 900) {
1574
  return { copied, copy }
1575
  }
1576
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1577
  function ChatState({
1578
  messages,
1579
  setLightbox,
1580
  endRef,
1581
  mode,
1582
  onNewConversation,
 
1583
  onRetryMessage,
1584
  chatSettings,
1585
  onUpdateChatSettings,
@@ -1598,6 +1698,9 @@ function ChatState({
1598
  endRef: React.RefObject<HTMLDivElement>
1599
  mode: Mode
1600
  onNewConversation: () => void
 
 
 
1601
  onRetryMessage: (id: string) => void
1602
  chatSettings: ChatScopedSettings
1603
  onUpdateChatSettings: (next: ChatScopedSettings) => void
@@ -1613,6 +1716,9 @@ function ChatState({
1613
  }) {
1614
  const { copied, copy } = useCopyMessage()
1615
  const [chatSettingsOpen, setChatSettingsOpen] = useState(false)
 
 
 
1616
  // Local overrides for view-pack angle switching (keyed by message id)
1617
  const [angleOverrides, setAngleOverrides] = useState<Record<string, { angle: string; url: string }>>({})
1618
 
@@ -1650,9 +1756,24 @@ function ChatState({
1650
 
1651
  return (
1652
  <div className="flex flex-col h-full w-full max-w-[52rem] mx-auto">
1653
- {/* Top-right fixed: Chat settings gear + New Chat icon (Phase 2, additive) */}
 
 
 
 
1654
  <div className="fixed top-3 right-5 z-50">
1655
- <div className="relative flex items-center gap-3">
 
 
 
 
 
 
 
 
 
 
 
1656
  <button
1657
  type="button"
1658
  onClick={() => setChatSettingsOpen((v) => !v)}
@@ -1701,6 +1822,20 @@ function ChatState({
1701
  key={m.id}
1702
  className={`flex gap-5 ${m.role === 'user' ? 'justify-end' : 'justify-start'}`}
1703
  >
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1704
  {m.role === 'assistant' ? (
1705
  <div className="w-8 h-8 rounded-full bg-white text-black flex items-center justify-center flex-shrink-0 font-bold text-sm mt-1">
1706
  /
@@ -1858,6 +1993,7 @@ function ChatState({
1858
  ) : null}
1859
  </div>
1860
  )}
 
1861
  </div>
1862
  ))}
1863
  <div ref={endRef} className="h-6" />
@@ -2040,6 +2176,22 @@ export default function App() {
2040
  })
2041
  const [voiceConversationId, setVoiceConversationId] = useState<string>(() => uuid())
2042
  const [lightbox, setLightbox] = useState<string | null>(null)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2043
 
2044
  // Persona Integration β€” "Save as Persona Avatar" from AvatarStudio (additive)
2045
  const [saveAsPersonaData, setSaveAsPersonaData] = useState<{ item: GalleryItem; outfits: GalleryItem[]; batchSiblings?: GalleryItem[] } | null>(null)
@@ -2311,6 +2463,26 @@ export default function App() {
2311
  // Track last spoken message to avoid re-speaking
2312
  const lastSpokenMessageIdRef = useRef<string | null>(null)
2313
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2314
  // When a voice session is explicitly chosen from Session Hub, skip the auto-link resolve
2315
  // to prevent the useEffect from overwriting the chosen session with the active one.
2316
  const voiceSessionExplicitRef = useRef(false)
@@ -3985,10 +4157,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
3985
  [authHeaders, conversationId, messages, settings.backendUrl, settings.funMode, settingsDraft]
3986
  )
3987
 
3988
- // TTS for assistant responses (speak-once pattern) - ONLY IN VOICE MODE
 
 
 
 
 
 
 
 
 
 
 
 
3989
  useEffect(() => {
3990
- // Only enable TTS when in Voice mode
3991
- if (mode !== 'voice') return
3992
 
3993
  const ttsEnabled = settingsDraft.ttsEnabled ?? true
3994
  if (!ttsEnabled || !window.SpeechService) return
@@ -4001,6 +4184,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
4001
  lastMessage.role === 'assistant' &&
4002
  !lastMessage.pending &&
4003
  lastMessage.text &&
 
4004
  lastMessage.id !== lastSpokenMessageIdRef.current
4005
  ) {
4006
  // Mark this message as spoken
@@ -4009,9 +4193,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
4009
  // Speak the assistant's response β€” strip markdown images/links so TTS
4010
  // doesn't read raw URLs aloud. Images are shown visually instead.
4011
  const speechText = stripMarkdownForSpeech(lastMessage.text)
4012
- if (speechText) window.SpeechService.speak(speechText)
 
 
 
 
 
 
 
 
 
 
 
 
4013
  }
4014
- }, [messages, settingsDraft.ttsEnabled, mode])
4015
 
4016
  const uploadAndSend = useCallback(
4017
  async (file: File) => {
@@ -4489,7 +4685,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
4489
 
4490
  {/* Top-right Project Indicator - Only shown in chat mode when a project is active */}
4491
  {mode === 'chat' && (
4492
- <header className="absolute top-0 left-0 right-0 pr-[7rem] pl-5 py-3 z-20 flex items-center justify-end gap-3 pointer-events-none">
4493
  {/* Project Indicator */}
4494
  {(() => {
4495
  const currentProjectId = localStorage.getItem('homepilot_current_project')
@@ -5049,6 +5245,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
5049
  endRef={endRef}
5050
  mode={mode}
5051
  onNewConversation={onNewConversation}
 
5052
  onRetryMessage={retryFailedMessage}
5053
  chatSettings={chatSettings}
5054
  onUpdateChatSettings={updateChatSettings}
@@ -5124,6 +5321,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
5124
  endRef={endRef}
5125
  mode={mode}
5126
  onNewConversation={onNewConversation}
 
5127
  onRetryMessage={retryFailedMessage}
5128
  chatSettings={chatSettings}
5129
  onUpdateChatSettings={updateChatSettings}
@@ -5149,6 +5347,91 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm.
5149
  />
5150
  ) : null}
5151
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5152
  {/* Save as Persona Avatar modal (from AvatarStudio gallery) */}
5153
  {saveAsPersonaData && (
5154
  <SaveAsPersonaModal
 
21
  Copy,
22
  RotateCw,
23
  PenLine,
24
+ Phone,
25
  Users,
26
  EyeOff,
27
  BookOpen,
 
35
  import AccountMenu, { type AccountMenuUser } from './components/AccountMenu'
36
  import { useAuth } from './components/AuthGate'
37
  import VoiceMode, { stripMarkdownForSpeech } from './VoiceModeGrok'
38
+ import CallOverlay from './CallOverlay'
39
+ import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log'
40
+ import PostCallCard from './phone/PostCallCard'
41
  // Legacy voice mode available as: import VoiceModeLegacy from './VoiceModeLegacy'
42
  import ProjectsView from './ProjectsView'
43
  import ImagineView from './Imagine'
 
117
  intent: 'generate_images' | 'generate_videos'
118
  prompt: string
119
  }
120
+ // "Call memory card" β€” when present the message renders as an
121
+ // inline record of a just-ended voice call instead of a regular
122
+ // bubble. Additive: untouched by every existing code path.
123
+ callMemory?: {
124
+ durationSec: number
125
+ /** Epoch-ms timestamp of when the call ended. Rendered as an
126
+ * auditable stamp (e.g. "7:36 PM") in the card header. */
127
+ endedAt?: number
128
+ /** Persona the user was speaking with during this call β€” feeds
129
+ * PostCallCard's "Call with X" / "X remembers" headers. */
130
+ personaName?: string
131
+ /** Optional transcript the user can expand inline from the card.
132
+ * Populated by App.tsx at call-end by slicing chatMessages
133
+ * between the call's open + close timestamps. */
134
+ transcript?: Array<{ who: 'user' | 'assistant'; text: string }>
135
+ }
136
  }
137
 
138
  // ---------------------------------------------------------------------------
 
1594
  return { copied, copy }
1595
  }
1596
 
1597
+ /**
1598
+ * CallMemoryCard β€” enterprise-style inline record of a just-ended
1599
+ * call. Rendered inside the chat stream as a neutral system event
1600
+ * rather than a floating toast or an emotional "moment" card. The
1601
+ * tone is auditable, not expressive: the card states what happened,
1602
+ * when, and offers one standard system action to resume.
1603
+ */
1604
+ function CallMemoryCard({
1605
+ durationSec,
1606
+ endedAt,
1607
+ onSpeakAgain,
1608
+ }: {
1609
+ durationSec: number
1610
+ endedAt?: number
1611
+ onSpeakAgain?: () => void
1612
+ }) {
1613
+ const fmt = (s: number) => {
1614
+ if (s < 60) return `${s}s`
1615
+ const m = Math.floor(s / 60)
1616
+ const r = s % 60
1617
+ return r === 0 ? `${m}m` : `${m}m ${r}s`
1618
+ }
1619
+ const stamp = endedAt
1620
+ ? new Date(endedAt).toLocaleTimeString(undefined, {
1621
+ hour: 'numeric', minute: '2-digit',
1622
+ })
1623
+ : ''
1624
+ return (
1625
+ <div
1626
+ className="flex items-start gap-5 w-full hp-fade-in"
1627
+ role="note"
1628
+ aria-label="Call record"
1629
+ >
1630
+ <div className="w-8 h-8 rounded-full bg-white/5 border border-white/10 text-white/50 flex items-center justify-center flex-shrink-0 mt-1">
1631
+ <Phone size={12} />
1632
+ </div>
1633
+ <div
1634
+ className="max-w-[85%] rounded-[12px] px-4 py-3 bg-white/[0.03] border border-white/10"
1635
+ >
1636
+ {/* Meta row β€” factual header, low emphasis */}
1637
+ <div className="flex items-center gap-2 text-[11px] text-white/50">
1638
+ <span>Call</span>
1639
+ <span className="text-white/25">Β·</span>
1640
+ <span
1641
+ className="font-mono tabular-nums text-white/70"
1642
+ style={{ fontFeatureSettings: '"tnum"' }}
1643
+ >
1644
+ {fmt(durationSec)}
1645
+ </span>
1646
+ {stamp ? (
1647
+ <>
1648
+ <span className="text-white/25">Β·</span>
1649
+ <span className="font-mono tabular-nums text-white/50">{stamp}</span>
1650
+ </>
1651
+ ) : null}
1652
+ </div>
1653
+ {/* Body β€” neutral factual line, no emotional copy */}
1654
+ <div className="mt-1.5 text-[13.5px] leading-snug text-white/75">
1655
+ Voice session completed
1656
+ </div>
1657
+ {/* CTA β€” system-style outline button, same weight as header icons */}
1658
+ <div className="mt-3">
1659
+ <button
1660
+ type="button"
1661
+ onClick={onSpeakAgain}
1662
+ disabled={!onSpeakAgain}
1663
+ className="inline-flex items-center gap-1.5 rounded-md bg-transparent hover:bg-white/5 border border-white/15 hover:border-white/25 text-white/80 hover:text-white px-3 py-1 text-[12px] font-medium transition-colors disabled:opacity-40 disabled:cursor-not-allowed"
1664
+ aria-label="Resume call"
1665
+ >
1666
+ <Phone size={12} />
1667
+ Resume call
1668
+ </button>
1669
+ </div>
1670
+ </div>
1671
+ </div>
1672
+ )
1673
+ }
1674
+
1675
+
1676
  function ChatState({
1677
  messages,
1678
  setLightbox,
1679
  endRef,
1680
  mode,
1681
  onNewConversation,
1682
+ onStartCall,
1683
  onRetryMessage,
1684
  chatSettings,
1685
  onUpdateChatSettings,
 
1698
  endRef: React.RefObject<HTMLDivElement>
1699
  mode: Mode
1700
  onNewConversation: () => void
1701
+ /** Primary call action β€” opens the live-voice mode. Optional so
1702
+ * callers that don't want the button can omit it. */
1703
+ onStartCall?: () => void
1704
  onRetryMessage: (id: string) => void
1705
  chatSettings: ChatScopedSettings
1706
  onUpdateChatSettings: (next: ChatScopedSettings) => void
 
1716
  }) {
1717
  const { copied, copy } = useCopyMessage()
1718
  const [chatSettingsOpen, setChatSettingsOpen] = useState(false)
1719
+ const handleStartCall = useCallback(() => {
1720
+ onStartCall?.()
1721
+ }, [onStartCall])
1722
  // Local overrides for view-pack angle switching (keyed by message id)
1723
  const [angleOverrides, setAngleOverrides] = useState<Record<string, { angle: string; url: string }>>({})
1724
 
 
1756
 
1757
  return (
1758
  <div className="flex flex-col h-full w-full max-w-[52rem] mx-auto">
1759
+ {/* Top-right fixed: a single flat group of neutral header icons.
1760
+ The call button is intentionally *not* a CTA β€” it sits inline
1761
+ with the other utility icons and only opens a dedicated call
1762
+ overlay (see CallOverlay). Call mode is a distinct session,
1763
+ not a styling of the header. */}
1764
  <div className="fixed top-3 right-5 z-50">
1765
+ <div className="relative flex items-center gap-2">
1766
+ {onStartCall && (
1767
+ <button
1768
+ type="button"
1769
+ onClick={handleStartCall}
1770
+ className="w-9 h-9 flex items-center justify-center rounded-full bg-white/5 border border-white/10 text-white/60 hover:bg-white/10 hover:text-white transition-colors"
1771
+ title="Start call"
1772
+ aria-label="Start call"
1773
+ >
1774
+ <Phone size={16} />
1775
+ </button>
1776
+ )}
1777
  <button
1778
  type="button"
1779
  onClick={() => setChatSettingsOpen((v) => !v)}
 
1822
  key={m.id}
1823
  className={`flex gap-5 ${m.role === 'user' ? 'justify-end' : 'justify-start'}`}
1824
  >
1825
+ {/* Inline "call memory card" β€” renders instead of a bubble
1826
+ when this message represents a just-ended voice call.
1827
+ Treated as assistant-aligned (left) so it reads as "a
1828
+ moment we shared" rather than a system toast. */}
1829
+ {m.callMemory ? (
1830
+ <PostCallCard
1831
+ durationSec={m.callMemory.durationSec}
1832
+ endedAt={m.callMemory.endedAt}
1833
+ personaName={m.callMemory.personaName || 'Assistant'}
1834
+ variant="expand"
1835
+ transcript={m.callMemory.transcript}
1836
+ onResume={onStartCall}
1837
+ />
1838
+ ) : (<>
1839
  {m.role === 'assistant' ? (
1840
  <div className="w-8 h-8 rounded-full bg-white text-black flex items-center justify-center flex-shrink-0 font-bold text-sm mt-1">
1841
  /
 
1993
  ) : null}
1994
  </div>
1995
  )}
1996
+ </>)}
1997
  </div>
1998
  ))}
1999
  <div ref={endRef} className="h-6" />
 
2176
  })
2177
  const [voiceConversationId, setVoiceConversationId] = useState<string>(() => uuid())
2178
  const [lightbox, setLightbox] = useState<string | null>(null)
2179
+ // Call mode β€” a dedicated session distinct from Voice mode. The
2180
+ // header πŸ“ž opens a centered CallOverlay; it does not swap modes.
2181
+ const [callOpen, setCallOpen] = useState(false)
2182
+ // `skipDialing` is flipped on for the "Speak again" re-entry so the
2183
+ // user doesn't have to sit through "calling…" twice in a row.
2184
+ const [callSkipDialing, setCallSkipDialing] = useState(false)
2185
+ // Snapshot of chatMessages.length at the moment a call opens β€”
2186
+ // used at call-end to slice the transcript of turns that happened
2187
+ // during the call, for PostCallCard's expandable transcript.
2188
+ const callOpenMsgIndexRef = useRef<number>(-1)
2189
+ useEffect(() => {
2190
+ if (callOpen) {
2191
+ callOpenMsgIndexRef.current = chatMessages.length
2192
+ }
2193
+ }, [callOpen]) // intentionally miss chatMessages β€” we want the
2194
+ // snapshot at OPEN time, not every keystroke after
2195
 
2196
  // Persona Integration β€” "Save as Persona Avatar" from AvatarStudio (additive)
2197
  const [saveAsPersonaData, setSaveAsPersonaData] = useState<{ item: GalleryItem; outfits: GalleryItem[]; batchSiblings?: GalleryItem[] } | null>(null)
 
2463
  // Track last spoken message to avoid re-speaking
2464
  const lastSpokenMessageIdRef = useRef<string | null>(null)
2465
 
2466
+ // Call-open history guard β€” when the overlay opens, seed the
2467
+ // last-spoken pointer to the CURRENT tail of ``messages``. Without
2468
+ // this the TTS-on-assistant-reply effect treats the previous
2469
+ // chat-history tail as a fresh reply the moment ``callOpen`` flips
2470
+ // true, producing a pre-existing message read-aloud that collides
2471
+ // with CallOverlay's own fallback opener (the "two openings
2472
+ // overlapping" symptom). Only NEW assistant messages that land
2473
+ // DURING the call will be spoken by App.tsx from here on; the
2474
+ // call-itself-opener is owned by CallOverlay via speakOwned().
2475
+ useEffect(() => {
2476
+ if (!callOpen) return
2477
+ const tail = messages[messages.length - 1]
2478
+ lastSpokenMessageIdRef.current = tail ? tail.id : null
2479
+ // Intentionally single-shot on callOpen flip β€” we don't want this
2480
+ // to re-run on every message mutation during the call, otherwise
2481
+ // the guard would swallow the very replies it's supposed to let
2482
+ // through.
2483
+ // eslint-disable-next-line react-hooks/exhaustive-deps
2484
+ }, [callOpen])
2485
+
2486
  // When a voice session is explicitly chosen from Session Hub, skip the auto-link resolve
2487
  // to prevent the useEffect from overwriting the chosen session with the active one.
2488
  const voiceSessionExplicitRef = useRef(false)
 
4157
  [authHeaders, conversationId, messages, settings.backendUrl, settings.funMode, settingsDraft]
4158
  )
4159
 
4160
+ // TTS for assistant responses (speak-once pattern). Fires when
4161
+ // the user is in Voice mode OR the CallOverlay is open. The
4162
+ // second branch matters in the chat-REST fallback path: when
4163
+ // VOICE_CALL_ENABLED is false server-side, the call runs through
4164
+ // the regular chat pipeline, assistant replies land in the
4165
+ // current ``messages`` array, and without this nothing speaks
4166
+ // them β€” the user hears silence during the call.
4167
+ //
4168
+ // SpeechService.speak + stripMarkdownForSpeech are the canonical
4169
+ // app-level TTS primitives; reusing them here keeps call-mode
4170
+ // TTS consistent with voice-mode TTS (same voice pick, same
4171
+ // markdown sanitation, same single-utterance dedupe via
4172
+ // lastSpokenMessageIdRef).
4173
  useEffect(() => {
4174
+ if (mode !== 'voice' && !callOpen) return
 
4175
 
4176
  const ttsEnabled = settingsDraft.ttsEnabled ?? true
4177
  if (!ttsEnabled || !window.SpeechService) return
 
4184
  lastMessage.role === 'assistant' &&
4185
  !lastMessage.pending &&
4186
  lastMessage.text &&
4187
+ !lastMessage.callMemory &&
4188
  lastMessage.id !== lastSpokenMessageIdRef.current
4189
  ) {
4190
  // Mark this message as spoken
 
4193
  // Speak the assistant's response β€” strip markdown images/links so TTS
4194
  // doesn't read raw URLs aloud. Images are shown visually instead.
4195
  const speechText = stripMarkdownForSpeech(lastMessage.text)
4196
+ if (!speechText) return
4197
+ // During a call, route through the shared speakOwned lock so
4198
+ // this app-level path can't double-speak alongside the overlay's
4199
+ // fallback opener (both targeting the same assistant message).
4200
+ // Outside a call, keep the legacy SpeechService.speak path β€”
4201
+ // voice-mode has no peer TTS source so the lock would add noise
4202
+ // without benefit.
4203
+ if (callOpen && isCallFullDuplexEnabled()) {
4204
+ clog({ e: 'turn', action: 'assistant_in', route: 'chat_rest' })
4205
+ speakOwned('app-level', speechText)
4206
+ } else {
4207
+ window.SpeechService.speak(speechText)
4208
+ }
4209
  }
4210
+ }, [messages, settingsDraft.ttsEnabled, mode, callOpen])
4211
 
4212
  const uploadAndSend = useCallback(
4213
  async (file: File) => {
 
4685
 
4686
  {/* Top-right Project Indicator - Only shown in chat mode when a project is active */}
4687
  {mode === 'chat' && (
4688
+ <header className="absolute top-0 left-0 right-0 pr-[9rem] pl-5 py-3 z-20 flex items-center justify-end gap-3 pointer-events-none">
4689
  {/* Project Indicator */}
4690
  {(() => {
4691
  const currentProjectId = localStorage.getItem('homepilot_current_project')
 
5245
  endRef={endRef}
5246
  mode={mode}
5247
  onNewConversation={onNewConversation}
5248
+ onStartCall={() => setCallOpen(true)}
5249
  onRetryMessage={retryFailedMessage}
5250
  chatSettings={chatSettings}
5251
  onUpdateChatSettings={updateChatSettings}
 
5321
  endRef={endRef}
5322
  mode={mode}
5323
  onNewConversation={onNewConversation}
5324
+ onStartCall={() => setCallOpen(true)}
5325
  onRetryMessage={retryFailedMessage}
5326
  chatSettings={chatSettings}
5327
  onUpdateChatSettings={updateChatSettings}
 
5347
  />
5348
  ) : null}
5349
 
5350
+ {/* Call overlay β€” a distinct session layered over the chat.
5351
+ Open it with the πŸ“ž header icon; end it with the red button
5352
+ inside the overlay. Not the same as Voice mode. */}
5353
+ <CallOverlay
5354
+ open={callOpen}
5355
+ onClose={() => {
5356
+ setCallOpen(false)
5357
+ // Reset the skip flag once the overlay has closed so the
5358
+ // next fresh tap on πŸ“ž starts with the full dial phase.
5359
+ setCallSkipDialing(false)
5360
+ }}
5361
+ skipDialing={callSkipDialing}
5362
+ onEnded={(durationSec) => {
5363
+ // Append an inline PostCallCard to the chat stream. Captures
5364
+ // the slice of turns exchanged during the call β€” which the
5365
+ // user can expand inline from the card (no modal) β€” plus
5366
+ // the persona name + end-of-call timestamp.
5367
+ const endedAt = Date.now()
5368
+ const personaName = currentProject?.name || 'Assistant'
5369
+ const startIdx = callOpenMsgIndexRef.current
5370
+ // Build a transcript only when we have a valid open-time
5371
+ // snapshot AND new messages landed during the call.
5372
+ let transcript:
5373
+ | Array<{ who: 'user' | 'assistant'; text: string }>
5374
+ | undefined
5375
+ if (startIdx >= 0 && startIdx <= chatMessages.length) {
5376
+ const slice = chatMessages.slice(startIdx)
5377
+ const lines = slice
5378
+ .filter((m) => !m.callMemory && (m.text || '').trim())
5379
+ .map((m) => ({
5380
+ who: (m.role === 'user' ? 'user' : 'assistant') as
5381
+ | 'user' | 'assistant',
5382
+ text: m.text.trim(),
5383
+ }))
5384
+ if (lines.length > 0) transcript = lines
5385
+ }
5386
+ setChatMessages((prev) => [
5387
+ ...prev,
5388
+ {
5389
+ id: `call-memory-${endedAt}`,
5390
+ role: 'assistant',
5391
+ text: '',
5392
+ callMemory: {
5393
+ durationSec,
5394
+ endedAt,
5395
+ personaName,
5396
+ transcript,
5397
+ },
5398
+ },
5399
+ ])
5400
+ }}
5401
+ personaName={currentProject?.name || 'Assistant'}
5402
+ messages={chatMessages}
5403
+ avatarUrl={(() => {
5404
+ // Resolve the persona's thumbnail the same way
5405
+ // PersonaSettingsPanel does: prefer
5406
+ // persona_appearance.selected_thumb_filename, then
5407
+ // .selected_filename, wrapped in the /files/ auth URL.
5408
+ const pap = (currentProject?.persona_appearance as Record<string, unknown> | undefined) || {}
5409
+ const rel =
5410
+ (pap.selected_thumb_filename as string | undefined) ||
5411
+ (pap.selected_filename as string | undefined)
5412
+ if (!rel) return null
5413
+ const base = settings.backendUrl.replace(/\/+$/, '')
5414
+ const tok = localStorage.getItem('homepilot_auth_token') || ''
5415
+ const clean = rel.replace(/^\/+/, '')
5416
+ return `${base}/files/${clean}${tok ? `?token=${encodeURIComponent(tok)}` : ''}`
5417
+ })()}
5418
+ onSendText={(text) => sendTextOrIntent(text)}
5419
+ backend={{
5420
+ backendUrl: settings.backendUrl,
5421
+ authToken: localStorage.getItem('homepilot_auth_token'),
5422
+ conversationId: chatConversationId,
5423
+ personaId: currentProject?.persona_agent
5424
+ ? (currentProject.persona_agent.id as string | undefined) || null
5425
+ : null,
5426
+ }}
5427
+ />
5428
+
5429
+ {/* Previous floating "call ended" toast has been replaced by an
5430
+ inline CallMemoryCard rendered directly in the chat stream
5431
+ (see the messages.map branch on m.callMemory). Keeping this
5432
+ comment as a breadcrumb for reviewers. */}
5433
+
5434
+
5435
  {/* Save as Persona Avatar modal (from AvatarStudio gallery) */}
5436
  {saveAsPersonaData && (
5437
  <SaveAsPersonaModal
frontend/src/ui/CallOverlay.tsx ADDED
@@ -0,0 +1,1389 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * CallOverlay.tsx β€” TSX port of the frontend/src/ui/phone/ design set.
3
+ *
4
+ * Maps 1:1 onto the design-only mockups in `phone/`:
5
+ * tokens.jsx β†’ HP_CALL + hpCallStateColor/Label (inlined below)
6
+ * icons.jsx β†’ <Icon* /> components
7
+ * controls.jsx β†’ <ControlBtn />
8
+ * avatar.jsx β†’ <CallAvatar />
9
+ * waveform.jsx β†’ <CallWaveform />
10
+ * call-modal.jsx β†’ <CallModal /> + the ModalPresentation wrapper
11
+ *
12
+ * Layered as a distinct "call mode" on top of the chat β€” it is NOT
13
+ * the same thing as Voice mode:
14
+ * 🎀 Voice = input method inside the chat
15
+ * πŸ“ž Call = a separate immersive session (this component)
16
+ *
17
+ * States (mirrors the designer's set):
18
+ * connecting Β· listening Β· thinking Β· speaking Β· muted
19
+ * (ended is intentionally short-lived β€” the overlay fades out.)
20
+ *
21
+ * This pass wires the visual shell. Real STT/TTS plumbing from
22
+ * useVoiceController maps `listening/thinking/speaking` later.
23
+ */
24
+ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
25
+ import { useVoiceController, type VoiceState } from './voice/useVoiceController'
26
+ import { useCallSession } from './call/useCallSession'
27
+ import Waveform, { type WaveformMode } from './phone/primitives/Waveform'
28
+ import Aura from './phone/primitives/Aura'
29
+ import { createStreamingTts, type StreamingTts } from './call/streamTts'
30
+ import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log'
31
+ import { useBargeInDetector } from './call/bargeIn'
32
+
33
+ export type CallState =
34
+ | 'dialing'
35
+ | 'connecting'
36
+ | 'listening'
37
+ | 'thinking'
38
+ | 'speaking'
39
+ | 'muted'
40
+ | 'ended'
41
+
42
+ // ── Staged lifecycle timings. Numbers chosen so the user *feels* the
43
+ // handshake instead of getting a hard screen flip: dial (ringing) β†’
44
+ // connecting β†’ listening (live). Dial is long enough for ~one full
45
+ // PSTN ringback cadence (2 s on / 4 s off in the US) so the user
46
+ // actually hears the phone "ring" before pickup.
47
+ const DIAL_MS = 2200 // πŸ“ž "calling…" β€” pulse-ring phase + one PSTN ring
48
+ const CONNECT_MS = 500 // πŸ”΅ "connecting…" β€” dots-pulse phase
49
+ const END_FADE_MS = 220 // πŸ”΄ end button β†’ modal fade-out β†’ onClose
50
+
51
+ // ── Design tokens (ported from phone/tokens.jsx) ──────────────────
52
+
53
+ const HP_CALL = {
54
+ font: 'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif',
55
+ fontTabular: 'ui-monospace, SFMono-Regular, "Roboto Mono", Menlo, monospace',
56
+ backdrop: 'rgba(5, 5, 6, 0.72)',
57
+ surface: '#0b0b0c',
58
+ surface2: '#121214',
59
+ border: 'rgba(255, 255, 255, 0.08)',
60
+ text: 'rgba(255, 255, 255, 0.92)',
61
+ text2: 'rgba(255, 255, 255, 0.55)',
62
+ text3: 'rgba(255, 255, 255, 0.35)',
63
+ accent: '#22d3ee',
64
+ stateListening: '#22d3ee',
65
+ stateThinking: '#a78bfa',
66
+ stateSpeaking: '#10b981',
67
+ stateError: '#f87171',
68
+ end: '#ef4444',
69
+ } as const
70
+
71
+ function hpCallStateColor(state: CallState): string {
72
+ switch (state) {
73
+ case 'listening': return HP_CALL.stateListening
74
+ case 'thinking': return HP_CALL.stateThinking
75
+ case 'speaking': return HP_CALL.stateSpeaking
76
+ case 'dialing':
77
+ case 'connecting': return HP_CALL.accent
78
+ case 'muted':
79
+ case 'ended': return HP_CALL.text3
80
+ default: return HP_CALL.text2
81
+ }
82
+ }
83
+
84
+ // Map the voice-controller's internal machine onto our UI machine.
85
+ // Returns null for states we don't want to reflect (OFF β€” overlay is
86
+ // open so "off" is not a meaningful UI frame; render last mapped).
87
+ function mapVoiceState(s: VoiceState): CallState | null {
88
+ switch (s) {
89
+ case 'LISTENING': return 'listening'
90
+ case 'THINKING': return 'thinking'
91
+ case 'SPEAKING': return 'speaking'
92
+ case 'IDLE': return 'listening'
93
+ case 'OFF': return null
94
+ default: return null
95
+ }
96
+ }
97
+
98
+ // Map the overlay's richer CallState onto the primitive Waveform's
99
+ // four-valued mode enum. The primitive doesn't know about dialing /
100
+ // connecting / thinking β€” those all fold to 'idle' (low ambient
101
+ // amplitude, faint colour); only the three audible states drive
102
+ // distinct waveform modes.
103
+ function waveformModeFromCallState(state: CallState): WaveformMode {
104
+ if (state === 'listening' || state === 'thinking') return 'listening'
105
+ if (state === 'speaking') return 'speaking'
106
+ if (state === 'muted') return 'muted'
107
+ return 'idle'
108
+ }
109
+
110
+ function hpCallStateLabel(state: CallState, personaName: string): string {
111
+ switch (state) {
112
+ case 'dialing': return `calling ${personaName.toLowerCase()}…`
113
+ case 'connecting': return 'connecting…'
114
+ case 'listening': return 'listening'
115
+ case 'thinking': return 'thinking'
116
+ case 'speaking': return 'speaking'
117
+ case 'muted': return 'microphone off'
118
+ case 'ended': return 'call ended'
119
+ default: return ''
120
+ }
121
+ }
122
+
123
+ // ── Icons (ported from phone/icons.jsx) ───────────────────────────
124
+
125
+ type IconProps = { size?: number; color?: string }
126
+ const IconBase: React.FC<IconProps & { strokeWidth?: number; children: React.ReactNode }> = ({
127
+ size = 20, color = 'currentColor', strokeWidth = 1.75, children,
128
+ }) => (
129
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
130
+ stroke={color} strokeWidth={strokeWidth}
131
+ strokeLinecap="round" strokeLinejoin="round">
132
+ {children}
133
+ </svg>
134
+ )
135
+ const IconPhoneEnd: React.FC<IconProps> = (p) => (
136
+ <IconBase {...p} strokeWidth={1.9}>
137
+ <path d="M4 14c5-5 11-5 16 0l-2 2-3-1v-2a9 9 0 0 0-6 0v2l-3 1-2-2z" transform="rotate(135 12 12)" />
138
+ </IconBase>
139
+ )
140
+ const IconMic: React.FC<IconProps> = (p) => (
141
+ <IconBase {...p}>
142
+ <rect x="9" y="3" width="6" height="12" rx="3" />
143
+ <path d="M5 11a7 7 0 0 0 14 0M12 18v3" />
144
+ </IconBase>
145
+ )
146
+ const IconMicOff: React.FC<IconProps> = (p) => (
147
+ <IconBase {...p}>
148
+ <path d="M9 9V6a3 3 0 0 1 6 0v5m0 4a3 3 0 0 1-6 0" />
149
+ <path d="M5 11a7 7 0 0 0 11.5 5.3M19 11a7 7 0 0 1-.4 2.3M12 18v3" />
150
+ <path d="M3 3l18 18" />
151
+ </IconBase>
152
+ )
153
+ const IconChat: React.FC<IconProps> = (p) => (
154
+ <IconBase {...p}>
155
+ <path d="M4 5h16v11H9l-5 4z" />
156
+ </IconBase>
157
+ )
158
+ const IconBack: React.FC<IconProps> = (p) => (
159
+ <IconBase {...p} strokeWidth={1.9}>
160
+ <path d="M15 5l-7 7 7 7" />
161
+ </IconBase>
162
+ )
163
+
164
+ // ── ControlBtn (ported from phone/controls.jsx) ───────────────────
165
+
166
+ type Tone = 'neutral' | 'danger' | 'start' | 'accent'
167
+ const ControlBtn: React.FC<{
168
+ size?: number
169
+ tone?: Tone
170
+ active?: boolean
171
+ disabled?: boolean
172
+ label?: string
173
+ ariaLabel?: string
174
+ onClick?: () => void
175
+ children: React.ReactNode
176
+ }> = ({
177
+ size = 48, tone = 'neutral', active = false, disabled, label, ariaLabel, onClick, children,
178
+ }) => {
179
+ const bg =
180
+ tone === 'danger' ? HP_CALL.end :
181
+ tone === 'start' ? HP_CALL.stateSpeaking :
182
+ tone === 'accent' ? HP_CALL.accent :
183
+ active ? 'rgba(255,255,255,0.92)' : HP_CALL.surface2
184
+ const fg =
185
+ tone === 'danger' || tone === 'start' ? '#ffffff' :
186
+ tone === 'accent' ? '#052c33' :
187
+ active ? '#0b0b0c' : HP_CALL.text
188
+ const glow =
189
+ tone === 'danger' ? 'rgba(239, 68, 68, 0.45)' :
190
+ tone === 'start' ? 'rgba(16, 185, 129, 0.45)' :
191
+ tone === 'accent' ? 'rgba(34, 211, 238, 0.4)' :
192
+ 'rgba(0, 0, 0, 0.3)'
193
+ const border = tone === 'neutral' && !active ? `1px solid ${HP_CALL.border}` : 'none'
194
+
195
+ return (
196
+ <div style={{
197
+ display: 'flex', flexDirection: 'column', alignItems: 'center', gap: 6,
198
+ opacity: disabled ? 0.45 : 1,
199
+ }}>
200
+ <button
201
+ type="button"
202
+ onClick={onClick}
203
+ disabled={disabled}
204
+ aria-label={ariaLabel || label || ''}
205
+ style={{
206
+ width: size, height: size, borderRadius: '50%',
207
+ background: bg, border, color: fg, padding: 0,
208
+ display: 'flex', alignItems: 'center', justifyContent: 'center',
209
+ cursor: disabled ? 'not-allowed' : 'pointer',
210
+ boxShadow:
211
+ tone === 'neutral' && !active
212
+ ? 'inset 0 1px 0 rgba(255,255,255,0.04)'
213
+ : `0 8px 22px ${glow}, inset 0 1px 0 rgba(255,255,255,0.12)`,
214
+ transition: 'transform 80ms ease, background 120ms ease, box-shadow 120ms ease',
215
+ }}
216
+ >
217
+ {children}
218
+ </button>
219
+ {label ? (
220
+ <span style={{
221
+ fontFamily: HP_CALL.font, fontSize: 11, fontWeight: 500, letterSpacing: 0.2,
222
+ color: HP_CALL.text3, textTransform: 'lowercase',
223
+ }}>{label}</span>
224
+ ) : null}
225
+ </div>
226
+ )
227
+ }
228
+
229
+ // ── CallAvatar (ported from phone/avatar.jsx) ─────────────────────
230
+
231
+ const CallAvatar: React.FC<{
232
+ size?: number
233
+ state: CallState
234
+ imageUrl?: string | null
235
+ accentColor?: string | null
236
+ }> = ({ size = 156, state, imageUrl = null, accentColor = null }) => {
237
+ const stateColor = hpCallStateColor(state)
238
+ const breathes =
239
+ state === 'listening' ||
240
+ state === 'connecting' ||
241
+ state === 'speaking' ||
242
+ state === 'dialing'
243
+ const showPulseRings = state === 'dialing'
244
+ const haloOuter = size + 28
245
+ // seed + hue derivation now live inside the Aura primitive
246
+ // (hashHue + moodOffset). This component only owns the halo +
247
+ // ring decorations that depend on CallState.
248
+
249
+ return (
250
+ <div style={{
251
+ position: 'relative', width: haloOuter, height: haloOuter,
252
+ display: 'flex', alignItems: 'center', justifyContent: 'center',
253
+ }}>
254
+ {/* Expanding pulse rings β€” only while dialing. Three rings on
255
+ staggered delays so the modal reads as "ringing", not just
256
+ "spinning up". */}
257
+ {showPulseRings && [0, 1, 2].map(i => (
258
+ <div key={i} aria-hidden="true" style={{
259
+ position: 'absolute', inset: 0, borderRadius: '50%',
260
+ border: `1.5px solid ${stateColor}`,
261
+ // Split to longhand so per-ring animationDelay doesn't race
262
+ // the shorthand reset β€” React warns when both are set
263
+ // (mixing shorthand + longhand produces inconsistent results
264
+ // across browsers).
265
+ animationName: 'hp-call-pulse-ring',
266
+ animationDuration: '1600ms',
267
+ animationTimingFunction: 'ease-out',
268
+ animationIterationCount: 'infinite',
269
+ animationDelay: `${i * 420}ms`,
270
+ opacity: 0,
271
+ }} />
272
+ ))}
273
+ <div style={{
274
+ position: 'absolute', inset: 0, borderRadius: '50%',
275
+ background: `radial-gradient(circle, ${stateColor}55 0%, transparent 68%)`,
276
+ filter: 'blur(14px)',
277
+ opacity: breathes ? 0.8 : 0.35,
278
+ animation: breathes ? 'hp-halo-breathe 2s ease-in-out infinite' : 'none',
279
+ }} />
280
+ <div style={{
281
+ position: 'absolute', width: size + 10, height: size + 10,
282
+ borderRadius: '50%',
283
+ border: `2px solid ${stateColor}`,
284
+ opacity: state === 'ended' || state === 'muted' ? 0.25 : 0.92,
285
+ }} />
286
+ <div
287
+ style={{
288
+ // ``state === 'ended'`` filter stays on this wrapper so
289
+ // the Aura primitive stays state-agnostic while the
290
+ // overlay still gets a visible "call's over" cue.
291
+ filter: state === 'ended'
292
+ ? 'grayscale(0.6) brightness(0.7)'
293
+ : 'none',
294
+ borderRadius: '50%',
295
+ }}
296
+ >
297
+ <Aura
298
+ seed={imageUrl || accentColor || 'homepilot'}
299
+ size={size}
300
+ photoUrl={imageUrl}
301
+ // Hue-drift polish is the only thing the Aura primitive
302
+ // animates internally. We disable it here β€” the overlay
303
+ // already owns the halo breath + dialing rings, and the
304
+ // combined motion was too busy on the call surface.
305
+ animated={false}
306
+ />
307
+ </div>
308
+ </div>
309
+ )
310
+ }
311
+
312
+ // CallWaveform used to live here β€” extracted to
313
+ // frontend/src/ui/phone/primitives/Waveform.tsx. Consumed via the
314
+ // imported <Waveform /> component + waveformModeFromCallState
315
+ // mapper above.
316
+
317
+ // ── CallModal (ported from phone/call-modal.jsx) ──────────────────
318
+
319
+ interface CallModalProps {
320
+ state: CallState
321
+ personaName: string
322
+ imageUrl?: string | null
323
+ accentColor?: string | null
324
+ durationSec: number
325
+ onEnd: () => void
326
+ onToggleMute: () => void
327
+ onMinimize: () => void
328
+ onSwitchToChat?: () => void
329
+ /** Live 0..1 intensity. Drives the waveform bars per frame. */
330
+ intensityRef?: React.MutableRefObject<number>
331
+ }
332
+
333
+ const CallModal: React.FC<CallModalProps> = ({
334
+ state, personaName, imageUrl = null, accentColor = null,
335
+ durationSec, onEnd, onToggleMute, onMinimize, onSwitchToChat,
336
+ intensityRef,
337
+ }) => {
338
+ const stateColor = hpCallStateColor(state)
339
+ const stateLabel = hpCallStateLabel(state, personaName)
340
+ const mm = Math.floor(durationSec / 60).toString().padStart(2, '0')
341
+ const ss = (durationSec % 60).toString().padStart(2, '0')
342
+ const preConnect = state === 'dialing' || state === 'connecting'
343
+ const timer = preConnect ? 'β€”:β€”' : `${mm}:${ss}`
344
+
345
+ return (
346
+ <div style={{
347
+ width: 'min(420px, 92vw)',
348
+ padding: '18px 22px 26px',
349
+ borderRadius: 24,
350
+ background: HP_CALL.surface,
351
+ border: `1px solid ${HP_CALL.border}`,
352
+ boxShadow:
353
+ '0 30px 80px rgba(0,0,0,0.55), 0 0 0 1px rgba(0,0,0,0.4), inset 0 1px 0 rgba(255,255,255,0.04)',
354
+ position: 'relative', overflow: 'hidden',
355
+ fontFamily: HP_CALL.font, color: HP_CALL.text,
356
+ animation: 'hp-call-in 180ms ease-out',
357
+ }}>
358
+ <div style={{
359
+ position: 'absolute', inset: '-40% -10% auto -10%', height: '60%',
360
+ background: `radial-gradient(ellipse at 50% 100%, ${stateColor}26 0%, transparent 65%)`,
361
+ pointerEvents: 'none', opacity: state === 'muted' ? 0.1 : 0.35,
362
+ }} />
363
+
364
+ {/* Header row */}
365
+ <div style={{
366
+ position: 'relative', zIndex: 2,
367
+ display: 'flex', alignItems: 'center',
368
+ height: 32, marginBottom: 10,
369
+ }}>
370
+ <button
371
+ type="button"
372
+ onClick={onMinimize}
373
+ aria-label="Minimize call"
374
+ style={{
375
+ width: 32, height: 32, borderRadius: 10, padding: 0,
376
+ background: 'transparent', border: 'none', cursor: 'pointer',
377
+ color: HP_CALL.text2,
378
+ display: 'flex', alignItems: 'center', justifyContent: 'center',
379
+ }}
380
+ >
381
+ <IconBack size={18} color={HP_CALL.text2} />
382
+ </button>
383
+ <div style={{
384
+ position: 'absolute', left: 40, right: 40, textAlign: 'center',
385
+ fontSize: 16, fontWeight: 600, letterSpacing: -0.1,
386
+ color: HP_CALL.text,
387
+ whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis',
388
+ pointerEvents: 'none',
389
+ }}>{personaName}</div>
390
+ </div>
391
+
392
+ {/* Timer */}
393
+ <div style={{
394
+ position: 'relative', zIndex: 2, textAlign: 'center',
395
+ fontSize: 13, color: HP_CALL.text2,
396
+ fontFamily: HP_CALL.fontTabular,
397
+ fontVariantNumeric: 'tabular-nums',
398
+ marginBottom: 18,
399
+ }}>{timer}</div>
400
+
401
+ {/* Avatar + halo */}
402
+ <div style={{
403
+ position: 'relative', zIndex: 2,
404
+ display: 'flex', justifyContent: 'center', marginBottom: 18,
405
+ }}>
406
+ <CallAvatar size={156} state={state} imageUrl={imageUrl} accentColor={accentColor} />
407
+ </div>
408
+
409
+ {/* State label + waveform/dots */}
410
+ <div style={{
411
+ position: 'relative', zIndex: 2,
412
+ display: 'flex', flexDirection: 'column', alignItems: 'center',
413
+ gap: 10, marginBottom: 22, minHeight: 52,
414
+ }}>
415
+ {preConnect ? (
416
+ <div style={{ display: 'flex', gap: 5 }}>
417
+ {[0, 1, 2].map(i => (
418
+ <div key={i} style={{
419
+ width: 5, height: 5, borderRadius: '50%',
420
+ background: HP_CALL.text,
421
+ opacity: 0.35 + i * 0.2,
422
+ // Longhand β€” keeps per-dot animationDelay from being
423
+ // reset by the shorthand during React rerenders.
424
+ animationName: 'hp-dot-pulse',
425
+ animationDuration: state === 'dialing' ? '1.2s' : '0.8s',
426
+ animationTimingFunction: 'ease-in-out',
427
+ animationIterationCount: 'infinite',
428
+ animationDelay: `${i * 0.12}s`,
429
+ }} />
430
+ ))}
431
+ </div>
432
+ ) : (
433
+ <Waveform
434
+ bars={26}
435
+ height={24}
436
+ mode={waveformModeFromCallState(state)}
437
+ seed={personaName}
438
+ intensityRef={intensityRef}
439
+ />
440
+ )}
441
+ <div style={{
442
+ fontSize: 14, fontWeight: 500, letterSpacing: 0.3,
443
+ color: stateColor, textTransform: 'lowercase',
444
+ }}>{stateLabel}</div>
445
+ </div>
446
+
447
+ {/* Three-button dock */}
448
+ <div style={{
449
+ position: 'relative', zIndex: 2,
450
+ display: 'flex', justifyContent: 'center', alignItems: 'center',
451
+ gap: 28,
452
+ }}>
453
+ <ControlBtn
454
+ size={56}
455
+ active={state === 'muted'}
456
+ ariaLabel={state === 'muted' ? 'Unmute microphone' : 'Mute microphone'}
457
+ onClick={onToggleMute}
458
+ disabled={state === 'dialing' || state === 'connecting' || state === 'ended'}
459
+ >
460
+ {state === 'muted'
461
+ ? <IconMicOff size={22} color="#0b0b0c" />
462
+ : <IconMic size={22} />}
463
+ </ControlBtn>
464
+
465
+ <ControlBtn
466
+ size={72} tone="danger" ariaLabel="End call"
467
+ onClick={onEnd}
468
+ >
469
+ <IconPhoneEnd size={26} color="#ffffff" />
470
+ </ControlBtn>
471
+
472
+ <ControlBtn
473
+ size={56} ariaLabel="Switch to text chat"
474
+ onClick={onSwitchToChat}
475
+ disabled={!onSwitchToChat || state === 'dialing' || state === 'connecting' || state === 'ended'}
476
+ >
477
+ <IconChat size={22} />
478
+ </ControlBtn>
479
+ </div>
480
+ </div>
481
+ )
482
+ }
483
+
484
+ // ── Public overlay component ─────────────────────────────────────
485
+
486
+ export interface CallOverlayProps {
487
+ open: boolean
488
+ onClose: () => void
489
+ /** Optional: the persona the user is calling (display only). */
490
+ personaName?: string
491
+ /** Optional: avatar URL for the orb. */
492
+ avatarUrl?: string | null
493
+ /** Optional: per-persona accent for the fallback aura. */
494
+ accentColor?: string | null
495
+ /** Optional: minimize handler β€” reserved for the future PIP. */
496
+ onMinimize?: () => void
497
+ /** Optional: "Switch to text chat" handler. */
498
+ onSwitchToChat?: () => void
499
+ /** Skip the dial-ring phase. Used by "Speak again" re-entry so the
500
+ * user doesn't have to sit through "calling…" twice in a row. */
501
+ skipDialing?: boolean
502
+ /** Fired when the call ends; receives the live-phase duration in
503
+ * seconds. Used by App.tsx to show the "call ended Β· Ns" toast. */
504
+ onEnded?: (durationSec: number) => void
505
+ /** Route a captured user utterance into the chat pipeline. When
506
+ * omitted the call runs in visual-only mode (no STT hook mounts).
507
+ * Only used as a fallback when the backend voice_call WS is
508
+ * unavailable (VOICE_CALL_ENABLED=false). */
509
+ onSendText?: (text: string) => void
510
+ /** When true the assistant is currently producing its reply (LLM
511
+ * is thinking). Maps CallState β†’ 'thinking'. Reserved β€” the
512
+ * voice controller surfaces this itself once TTS events land. */
513
+ isAssistantThinking?: boolean
514
+ /** Backend config for the dedicated voice_call session. When
515
+ * supplied, the overlay will POST /v1/voice-call/sessions and
516
+ * open the WebSocket; if the backend responds with 404/501 we
517
+ * cleanly fall back to onSendText (chat REST). */
518
+ backend?: {
519
+ backendUrl: string
520
+ authToken: string | null
521
+ conversationId?: string | null
522
+ personaId?: string | null
523
+ }
524
+ /** Optional chat thread. When present AND the call runs in
525
+ * fallback mode (VOICE_CALL_ENABLED=false or unreachable),
526
+ * CallOverlay watches this array for new assistant replies
527
+ * that arrive DURING the call and speaks them via TTS β€” so
528
+ * the user hears the persona even without the voice_call WS.
529
+ *
530
+ * We also synthesize a short opener the moment the call
531
+ * connects in fallback mode, so the AI still "answers first"
532
+ * the way a real callee would. The baseline is captured at
533
+ * call-open so pre-existing chat history isn't re-spoken. */
534
+ messages?: ReadonlyArray<{
535
+ id: string
536
+ role: 'user' | 'assistant' | string
537
+ text: string
538
+ callMemory?: unknown
539
+ }>
540
+ }
541
+
542
+ // Outer shell β€” renders nothing when closed so the voice hook inside
543
+ // CallOverlayInner only mounts during an active call. This matters: the
544
+ // voice controller requests mic permission + starts the VAD as soon as
545
+ // it mounts. Keeping it gated behind `open` means the mic is only
546
+ // touched while the user is in a call.
547
+ export default function CallOverlay(props: CallOverlayProps) {
548
+ if (!props.open) return null
549
+ return <CallOverlayInner {...props} />
550
+ }
551
+
552
+ function CallOverlayInner({
553
+ onClose,
554
+ personaName = 'Assistant',
555
+ avatarUrl = null,
556
+ accentColor = null,
557
+ onMinimize,
558
+ onSwitchToChat,
559
+ skipDialing = false,
560
+ onEnded,
561
+ onSendText,
562
+ backend,
563
+ messages,
564
+ }: CallOverlayProps) {
565
+ const initial: CallState = skipDialing ? 'connecting' : 'dialing'
566
+ const [state, setState] = useState<CallState>(initial)
567
+ const [muted, setMuted] = useState(false)
568
+ const [durationSec, setDurationSec] = useState(0)
569
+
570
+ // Staged lifecycle on mount:
571
+ // dialing (DIAL_MS) β†’ connecting (CONNECT_MS) β†’ listening
572
+ // (or: connecting β†’ listening when skipDialing).
573
+ useEffect(() => {
574
+ const timers: number[] = []
575
+ if (!skipDialing) {
576
+ timers.push(window.setTimeout(() => {
577
+ setState(s => (s === 'dialing' ? 'connecting' : s))
578
+ }, DIAL_MS))
579
+ }
580
+ timers.push(window.setTimeout(() => {
581
+ setState(s =>
582
+ (s === 'connecting' || s === 'dialing') ? 'listening' : s
583
+ )
584
+ }, (skipDialing ? 0 : DIAL_MS) + CONNECT_MS))
585
+
586
+ return () => { timers.forEach(t => window.clearTimeout(t)) }
587
+ }, [skipDialing])
588
+
589
+ // Live timer during active call (dialing + connecting don't count).
590
+ useEffect(() => {
591
+ if (state === 'dialing' || state === 'connecting' || state === 'ended') return
592
+ const iv = window.setInterval(() => setDurationSec((n) => n + 1), 1000)
593
+ return () => window.clearInterval(iv)
594
+ }, [state])
595
+
596
+ // Ringback tone β€” PSTN-standard two-frequency synthesis via
597
+ // WebAudio. US telephony uses 440 Hz + 480 Hz played together, 2 s
598
+ // on / 4 s off. We play one full on-phase (2 s) during dialing so
599
+ // the user hears a real phone-style ring before pickup. Nothing
600
+ // plays during connecting or live β€” a phone that keeps ringing
601
+ // after pickup is a broken phone.
602
+ useEffect(() => {
603
+ if (state !== 'dialing') return
604
+ const AC =
605
+ (window as unknown as { AudioContext?: typeof AudioContext }).AudioContext ||
606
+ (window as unknown as { webkitAudioContext?: typeof AudioContext }).webkitAudioContext
607
+ if (!AC) return
608
+ let ctx: AudioContext | null = null
609
+ try { ctx = new AC() } catch { return }
610
+
611
+ const start = ctx.currentTime + 0.03
612
+ const durationSec = 2.0 // "on" portion of the 2-on-4-off cadence
613
+ const peakGain = 0.07 // loud enough to be heard, soft enough to sit
614
+ // under the persona's TTS once it lands
615
+
616
+ // 440 Hz + 480 Hz fundamentals β€” the exact CCITT-spec recipe for
617
+ // "ringing signal, North American" (Precise Tone Plan).
618
+ const freqs = [440, 480] as const
619
+ const oscs: OscillatorNode[] = []
620
+ const gain = ctx.createGain()
621
+ gain.gain.setValueAtTime(0, start)
622
+ gain.gain.linearRampToValueAtTime(peakGain, start + 0.08)
623
+ gain.gain.setValueAtTime(peakGain, start + durationSec - 0.2)
624
+ gain.gain.linearRampToValueAtTime(0, start + durationSec)
625
+ gain.connect(ctx.destination)
626
+
627
+ for (const f of freqs) {
628
+ const o = ctx.createOscillator()
629
+ o.type = 'sine'
630
+ o.frequency.setValueAtTime(f, start)
631
+ o.connect(gain)
632
+ o.start(start)
633
+ o.stop(start + durationSec + 0.05)
634
+ oscs.push(o)
635
+ }
636
+
637
+ return () => {
638
+ for (const o of oscs) { try { o.stop() } catch { /* ignore */ } }
639
+ try { ctx?.close() } catch { /* ignore */ }
640
+ }
641
+ }, [state])
642
+
643
+ const handleEnd = useCallback(() => {
644
+ setState('ended')
645
+ // If the backend session is live, send a graceful call.control
646
+ // 'end' envelope so the server-side ledger closes cleanly.
647
+ if (useBackendRef.current) {
648
+ try { sessionRef.current.end() } catch { /* ignore */ }
649
+ }
650
+ // Capture the live-phase duration NOW β€” durationSec is frozen at
651
+ // the last tick because the interval effect tears down on
652
+ // state==='ended'. Close after a short fade so the backdrop +
653
+ // modal get a chance to animate out.
654
+ const endedWith = durationSec
655
+ window.setTimeout(() => {
656
+ if (onEnded) onEnded(endedWith)
657
+ onClose()
658
+ }, END_FADE_MS)
659
+ }, [onClose, onEnded, durationSec])
660
+
661
+ // Esc ends.
662
+ useEffect(() => {
663
+ const onKey = (e: KeyboardEvent) => { if (e.key === 'Escape') handleEnd() }
664
+ window.addEventListener('keydown', onKey)
665
+ return () => window.removeEventListener('keydown', onKey)
666
+ }, [handleEnd])
667
+
668
+ // ── Backend voice_call session (preferred transport) ───────────
669
+ // When a `backend` prop is supplied we attempt the dedicated
670
+ // voice_call WebSocket. If the backend responds 404/501 we stay
671
+ // in 'unavailable' and the chat-REST fallback takes over below.
672
+ const session = useCallSession({
673
+ enabled: !!backend,
674
+ backendUrl: backend?.backendUrl ?? '',
675
+ authToken: backend?.authToken ?? null,
676
+ request: useMemo(() => ({
677
+ conversation_id: backend?.conversationId ?? null,
678
+ persona_id: backend?.personaId ?? null,
679
+ entry_mode: 'call' as const,
680
+ // Declare Phase 2/3 client capability. Server replies with
681
+ // its own capability advertisement; effective mode is the
682
+ // intersection (useCallSession reads capabilities from the
683
+ // session-create response).
684
+ device_info: {
685
+ tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
686
+ platform: navigator.platform,
687
+ streaming: true,
688
+ barge_in: true,
689
+ },
690
+ }), [backend?.conversationId, backend?.personaId]),
691
+ })
692
+
693
+ const useBackend =
694
+ !!backend &&
695
+ session.status !== 'unavailable' &&
696
+ session.status !== 'error'
697
+
698
+ // ── "Listen first" gate β€” opener-TTS is playing ───────────────
699
+ // When the backend emits the curated opening greeting as a
700
+ // transcript.final right after call.state live, the client's
701
+ // microphone must stay CLOSED until the greeting has finished
702
+ // speaking. Otherwise the user's natural "hello?" mid-greeting
703
+ // triggers a barge-in and cuts the persona off β€” the opposite
704
+ // of the "AI answers first, user listens" product spec.
705
+ //
706
+ // openerDoneRef is a one-shot β€” once the first assistant
707
+ // utterance has finished, subsequent utterances (responses to
708
+ // user turns) do NOT relock the mic; the user can barge-in on
709
+ // those freely, which is the correct UX.
710
+ const [openerPlaying, setOpenerPlaying] = useState(false)
711
+ // ``connected`` is a pure derivation from ``state``; declared early
712
+ // so effects that fire before the setHandsFree gate (fallback opener
713
+ // below) can reference it without a forward-ref.
714
+ const connected =
715
+ state !== 'dialing' && state !== 'connecting' && state !== 'ended'
716
+ const openerDoneRef = useRef(false)
717
+ useEffect(() => {
718
+ if (session.status === 'creating') {
719
+ openerDoneRef.current = false
720
+ setOpenerPlaying(false)
721
+ }
722
+ }, [session.status])
723
+
724
+ // ── Real voice pipeline ────────────────────────────────────────
725
+ // STT capture β†’ preferred transport. If the backend session is
726
+ // live we route transcripts through the voice_call WS; otherwise
727
+ // we fall back to onSendText (regular chat REST) so the overlay
728
+ // still works when VOICE_CALL_ENABLED is off server-side.
729
+ const onSendTextRef = useRef(onSendText)
730
+ useEffect(() => { onSendTextRef.current = onSendText }, [onSendText])
731
+ const sessionRef = useRef(session)
732
+ useEffect(() => { sessionRef.current = session }, [session])
733
+ const useBackendRef = useRef(useBackend)
734
+ useEffect(() => { useBackendRef.current = useBackend }, [useBackend])
735
+
736
+ const voice = useVoiceController((text: string) => {
737
+ clog({
738
+ e: 'turn',
739
+ action: 'user_out',
740
+ route: useBackendRef.current ? 'ws' : 'chat_rest',
741
+ })
742
+ if (useBackendRef.current) {
743
+ sessionRef.current.sendTranscript(text)
744
+ } else {
745
+ onSendTextRef.current?.(text)
746
+ }
747
+ })
748
+
749
+ // Speak assistant replies that land over the WS. (In the chat-REST
750
+ // fallback path, the existing assistant-message pipeline already
751
+ // triggers TTS β€” so this effect is a no-op there.)
752
+ //
753
+ // In STREAMING mode this stays a no-op for the assistant turn
754
+ // (partials feed streamTts below). It still fires for any
755
+ // transcript.final the server emits OUTSIDE a streamed turn β€”
756
+ // e.g. the opening greeting, which the current ws.py path sends
757
+ // as transcript.final regardless of streaming mode.
758
+ // Shared TTS helper β€” called by both the WS path AND the
759
+ // fallback-mode openers/replies below. The FIRST utterance on a
760
+ // mounted session is treated as the opener (locks the mic gate
761
+ // via openerPlaying / openerDoneRef); subsequent utterances play
762
+ // freely so the user can barge in.
763
+
764
+ // ── Turn-lock state machine (full-duplex fix) ─────────────────
765
+ // The overlay owns a simple 3-state machine:
766
+ // 'ai' β€” AI is speaking; VAD events from the mic must be
767
+ // suppressed (the AI's own voice bleeding through the
768
+ // laptop speaker would otherwise trigger a false user
769
+ // turn β€” this is the "AI says yes, hears itself, then
770
+ // stops" bug reported in the log).
771
+ // 'user' β€” user is actively speaking (voice.state === LISTENING)
772
+ // 'idle' β€” nobody has the floor; VAD drives transitions normally
773
+ //
774
+ // Transitions fire setListeningSuppressed on the voice controller
775
+ // so suppression is a single source of truth. 300 ms release
776
+ // margin on 'ai β†’ idle' lets echo tail die before we trust the
777
+ // mic again. Gated behind isCallFullDuplexEnabled() so the old
778
+ // behaviour is restorable via localStorage override.
779
+ type TurnLock = 'ai' | 'user' | 'idle'
780
+ const turnLockRef = useRef<TurnLock>('idle')
781
+ const turnUnlockTimerRef = useRef<number | null>(null)
782
+ const fullDuplexEnabledRef = useRef<boolean>(isCallFullDuplexEnabled())
783
+
784
+ const setTurnLock = useCallback((next: TurnLock, trigger: string) => {
785
+ if (!fullDuplexEnabledRef.current) return
786
+ const prev = turnLockRef.current
787
+ if (prev === next) return
788
+ turnLockRef.current = next
789
+ voice.setListeningSuppressed(next === 'ai', `turn_lock:${trigger}`)
790
+ clog({ e: 'state', from: prev, to: next, trigger })
791
+ }, [voice])
792
+
793
+ const releaseAiTurnWithMargin = useCallback((trigger: string) => {
794
+ if (turnUnlockTimerRef.current) {
795
+ window.clearTimeout(turnUnlockTimerRef.current)
796
+ turnUnlockTimerRef.current = null
797
+ }
798
+ turnUnlockTimerRef.current = window.setTimeout(() => {
799
+ setTurnLock('idle', `${trigger}:margin_release`)
800
+ turnUnlockTimerRef.current = null
801
+ }, 300)
802
+ }, [setTurnLock])
803
+
804
+ useEffect(() => {
805
+ return () => {
806
+ if (turnUnlockTimerRef.current) {
807
+ window.clearTimeout(turnUnlockTimerRef.current)
808
+ turnUnlockTimerRef.current = null
809
+ }
810
+ }
811
+ }, [])
812
+
813
+ const speakText = useCallback((text: string) => {
814
+ if (!text || !text.trim()) return
815
+ setTurnLock('ai', 'speak:start')
816
+ const isOpener = !openerDoneRef.current
817
+ if (isOpener) setOpenerPlaying(true)
818
+ const markDone = () => {
819
+ openerDoneRef.current = true
820
+ setOpenerPlaying(false)
821
+ // 300 ms margin before we flip the lock back to 'idle'. Echo
822
+ // tail can linger after TTS says "ended"; releasing immediately
823
+ // would let the mic re-open while our own voice is still in
824
+ // the air.
825
+ releaseAiTurnWithMargin('speak:end')
826
+ }
827
+ // Length-based estimate for the SpeechService path (the shim
828
+ // doesn't expose onend). ~12 chars/sec; floor 1.5 s so "Yes?"
829
+ // still gives the user a beat; ceiling 8 s so a runaway
830
+ // greeting can't lock the mic indefinitely.
831
+ const estimateSpeechMs = (t: string) =>
832
+ Math.max(1500, Math.min(8000, t.length * 80))
833
+ try {
834
+ // Prefer the owned TTS path when full-duplex is on. speakOwned
835
+ // handles both the window.SpeechService route AND the
836
+ // cross-source dedupe (see call/log.ts). Returns true iff this
837
+ // caller owns the utterance; false means another source
838
+ // (App-level TTS) was mid-speak β€” we fall through to the legacy
839
+ // speechSynthesis path unchanged so the overlay isn't silent.
840
+ if (fullDuplexEnabledRef.current) {
841
+ const spoke = speakOwned('overlay', text, {
842
+ onEnd: markDone,
843
+ onError: () => markDone(),
844
+ })
845
+ if (spoke) {
846
+ if (isOpener) window.setTimeout(markDone, estimateSpeechMs(text))
847
+ return
848
+ }
849
+ } else {
850
+ const w = window as unknown as {
851
+ SpeechService?: { speak?: (t: string) => void }
852
+ }
853
+ if (w.SpeechService?.speak) {
854
+ w.SpeechService.speak(text)
855
+ if (isOpener) window.setTimeout(markDone, estimateSpeechMs(text))
856
+ return
857
+ }
858
+ }
859
+ if ('speechSynthesis' in window) {
860
+ const utt = new SpeechSynthesisUtterance(text)
861
+ if (isOpener) {
862
+ utt.onend = markDone
863
+ utt.onerror = markDone
864
+ window.setTimeout(markDone, estimateSpeechMs(text))
865
+ }
866
+ window.speechSynthesis.speak(utt)
867
+ return
868
+ }
869
+ if (isOpener) markDone()
870
+ } catch {
871
+ if (isOpener) markDone()
872
+ }
873
+ }, [setTurnLock, releaseAiTurnWithMargin])
874
+
875
+ // WS path β€” speak every assistant transcript.final (the opening
876
+ // greeting from ws.py, and in unary mode every reply). In
877
+ // streaming mode, per-token partials go through streamTts below
878
+ // and THIS path fires only for transcript.final emitted outside
879
+ // a streamed turn β€” e.g. the opener itself.
880
+ useEffect(() => {
881
+ if (!useBackend) return
882
+ const unsub = session.onAssistantTranscript((p) => {
883
+ clog({ e: 'turn', action: 'assistant_in', route: 'ws' })
884
+ speakText(p.text)
885
+ })
886
+ return unsub
887
+ }, [useBackend, session, speakText])
888
+
889
+ // ── Fallback mode (VOICE_CALL_ENABLED=false on backend) ───────
890
+ // When the backend voice_call route isn't mounted, the session
891
+ // POST 404s, useBackend stays false, and the STT pipeline routes
892
+ // through the regular chat-REST path (onSendText above). That
893
+ // works for text round-tripping β€” but without the hooks below,
894
+ // the assistant's replies land in chatMessages and are never
895
+ // spoken, so the user hears nothing.
896
+ //
897
+ // These two effects close that gap so the overlay is functional
898
+ // even with the WS disabled:
899
+ // (a) a synthetic opener fires when the call connects, so the
900
+ // AI still "answers first"
901
+ // (b) every new assistant message arriving in ``messages``
902
+ // during the call is spoken via the same TTS path the WS
903
+ // reply path uses
904
+
905
+ // (a) Fallback opener β€” once per session, when the line connects
906
+ // and we're running fallback mode, pick a short greeting and
907
+ // speak it. The greeting is marked as opener via speakText's
908
+ // first-utterance heuristic, so the mic gate stays closed
909
+ // exactly like the WS path.
910
+ const fallbackOpeners = useMemo(
911
+ () => [
912
+ 'Hello?',
913
+ 'Yes?',
914
+ `Hi, this is ${personaName}.`,
915
+ `${personaName} speaking.`,
916
+ `Hey β€” ${personaName}.`,
917
+ `Hi, ${personaName} here.`,
918
+ ],
919
+ [personaName],
920
+ )
921
+ const fallbackOpenerFiredRef = useRef(false)
922
+ useEffect(() => {
923
+ if (useBackend) return
924
+ if (!connected) return
925
+ if (fallbackOpenerFiredRef.current) return
926
+ fallbackOpenerFiredRef.current = true
927
+ const g = fallbackOpeners[
928
+ Math.floor(Math.random() * fallbackOpeners.length)
929
+ ]
930
+ speakText(g)
931
+ }, [useBackend, connected, fallbackOpeners, speakText])
932
+
933
+ // (b) Ongoing assistant replies in fallback mode are now spoken
934
+ // by the App-level TTS effect (App.tsx β€” gated on callOpen). That
935
+ // pipeline already handles stripMarkdownForSpeech + per-message
936
+ // dedupe via lastSpokenMessageIdRef, so we don't duplicate it
937
+ // here. The fallback OPENER above stays β€” it's synthesized in
938
+ // the overlay and never enters chatMessages, so the App-level
939
+ // effect can't produce it.
940
+ //
941
+ // ``messages`` prop is still accepted for future use (mid-call
942
+ // transcript viewer, etc.) but intentionally unused here β€” no
943
+ // TTS side-effect reads it.
944
+
945
+ // ── Phase 2: streaming TTS pipeline ───────────────────────────
946
+ // Holds one streamTts instance per mounted overlay. Stopped on
947
+ // unmount so a re-entry creates a fresh engine.
948
+ const ttsRef = useRef<StreamingTts | null>(null)
949
+ useEffect(() => {
950
+ if (!useBackend || !session.streamingNegotiated) {
951
+ ttsRef.current = null
952
+ return
953
+ }
954
+ const tts = createStreamingTts()
955
+ ttsRef.current = tts
956
+ return () => {
957
+ try { tts.stop() } catch { /* ignore */ }
958
+ ttsRef.current = null
959
+ }
960
+ }, [useBackend, session.streamingNegotiated])
961
+
962
+ // Track the currently-streaming turn_id so the barge-in path can
963
+ // address the right turn without racing a new one.
964
+ const currentTurnIdRef = useRef<string | null>(null)
965
+
966
+ // Partial deltas β†’ streamTts.appendDelta. Each delta triggers
967
+ // sentence-boundary flushing inside the TTS engine; callers don't
968
+ // see the buffering.
969
+ useEffect(() => {
970
+ if (!useBackend || !session.streamingNegotiated) return
971
+ return session.onAssistantPartial((p) => {
972
+ currentTurnIdRef.current = p.turn_id
973
+ ttsRef.current?.appendDelta(p.delta)
974
+ })
975
+ }, [useBackend, session])
976
+
977
+ // Turn end β†’ flush residual buffer + clear the active turn id.
978
+ useEffect(() => {
979
+ if (!useBackend || !session.streamingNegotiated) return
980
+ return session.onAssistantTurnEnd((p) => {
981
+ if (p.reason === 'cancelled' || p.reason === 'error') {
982
+ ttsRef.current?.stop()
983
+ } else {
984
+ ttsRef.current?.flush()
985
+ }
986
+ if (currentTurnIdRef.current === p.turn_id) {
987
+ currentTurnIdRef.current = null
988
+ }
989
+ })
990
+ }, [useBackend, session])
991
+
992
+ // Server-initiated cancel (barge-in ack) β†’ stop TTS immediately.
993
+ // Must land inside ~50 ms of receipt (Β§ 5.3 contract).
994
+ useEffect(() => {
995
+ if (!useBackend || !session.streamingNegotiated) return
996
+ return session.onAssistantCancel((_p) => {
997
+ ttsRef.current?.stop()
998
+ })
999
+ }, [useBackend, session])
1000
+
1001
+ // ── Phase 3: barge-in VAD tap ─────────────────────────────────
1002
+ // Audio level mirror β€” useVoiceController returns fresh values
1003
+ // every render; we stash the latest into a ref so the rAF loop
1004
+ // inside useBargeInDetector always sees the current value.
1005
+ const voiceLevelRef = useRef(0)
1006
+ useEffect(() => {
1007
+ voiceLevelRef.current = voice.audioLevel
1008
+ }, [voice.audioLevel])
1009
+
1010
+ // Re-evaluated each render; the detector's rAF loop reads its own
1011
+ // refs so identity changes don't thrash.
1012
+ const bargeInEnabled =
1013
+ useBackend &&
1014
+ session.bargeInNegotiated &&
1015
+ (ttsRef.current?.isSpeaking ?? false)
1016
+
1017
+ useBargeInDetector({
1018
+ audioLevelRef: voiceLevelRef,
1019
+ enabled: bargeInEnabled,
1020
+ onBargeIn: () => {
1021
+ const tid = currentTurnIdRef.current
1022
+ clog({ e: 'turn', action: 'barge_in', route: 'ws' })
1023
+ ttsRef.current?.stop()
1024
+ if (tid) session.sendBargeIn(tid)
1025
+ },
1026
+ })
1027
+
1028
+ // Call-center-style auto-greeting β€” the moment the backend session
1029
+ // goes live, prompt the persona to answer first (the way a real
1030
+ // person picks up the phone). We send a short synthetic transcript
1031
+ // the backend's persona_call phase machine will treat as turn 0 of
1032
+ // the opening sequence β€” it responds with a contextual "hello?" /
1033
+ // "hey, what's up?" in the persona's own voice.
1034
+ //
1035
+ // This is a one-shot per call session. `greetedRef` guards against
1036
+ // reconnect storms triggering a second "hello".
1037
+ const greetedRef = useRef(false)
1038
+ useEffect(() => {
1039
+ if (!useBackend) return
1040
+ if (session.callState !== 'live') return
1041
+ if (greetedRef.current) return
1042
+ greetedRef.current = true
1043
+ // One very short open-channel ping. persona_call reads this as the
1044
+ // caller-initiated summons in the Schegloff opening structure and
1045
+ // the persona answers.
1046
+ session.sendTranscript('[phone-call-open]')
1047
+ }, [useBackend, session])
1048
+
1049
+ // Reset the greet guard when a fresh session is (re)created.
1050
+ useEffect(() => {
1051
+ if (session.status === 'creating') greetedRef.current = false
1052
+ }, [session.status])
1053
+
1054
+ // Backchannel + filler events are short, low-volume TTS clips.
1055
+ // We route them through the same speak path; they're short enough
1056
+ // (one token) that interrupting a reply mid-stream is fine.
1057
+ useEffect(() => {
1058
+ if (!useBackend) return
1059
+ const unsub1 = session.onAssistantBackchannel((p) => {
1060
+ try { window.speechSynthesis?.speak?.(new SpeechSynthesisUtterance(p.token)) }
1061
+ catch { /* ignore */ }
1062
+ })
1063
+ const unsub2 = session.onAssistantFiller((p) => {
1064
+ try { window.speechSynthesis?.speak?.(new SpeechSynthesisUtterance(p.token)) }
1065
+ catch { /* ignore */ }
1066
+ })
1067
+ return () => { unsub1(); unsub2() }
1068
+ }, [useBackend, session])
1069
+
1070
+ // Enter hands-free once the line is "connected" AND the opener
1071
+ // has finished speaking. The second gate is the "listen first"
1072
+ // rule β€” the AI speaks its greeting and the user hears it out
1073
+ // before the mic opens; otherwise a natural early "hello?" trips
1074
+ // the barge-in detector and cuts the persona off mid-greeting.
1075
+ // ``connected`` is declared earlier (near state).
1076
+ //
1077
+ // This effect intentionally has NO cleanup that toggles handsFree β€”
1078
+ // the original cleanup raced against `openerPlaying` transitions
1079
+ // and re-render-driven re-runs (voice + setTurnLock change identity
1080
+ // on every render), which flipped handsFree OFF mid-call and left
1081
+ // state on OFF right when the user started speaking. The STT path
1082
+ // then dropped the VAD speech_start because state !== IDLE. Tear-
1083
+ // down lives in the dedicated unmount effect below.
1084
+ const voiceHandleRef = useRef(voice)
1085
+ useEffect(() => { voiceHandleRef.current = voice }, [voice])
1086
+ const setTurnLockRef = useRef(setTurnLock)
1087
+ useEffect(() => { setTurnLockRef.current = setTurnLock }, [setTurnLock])
1088
+
1089
+ useEffect(() => {
1090
+ if (!connected) return
1091
+ if (openerPlaying) return
1092
+ setTurnLockRef.current('idle', 'connected_open_mic')
1093
+ voiceHandleRef.current.setHandsFree(true)
1094
+ }, [connected, openerPlaying])
1095
+
1096
+ // Unmount-only cleanup. Guarantees the mic is released and the
1097
+ // turn-lock is reset when the overlay actually tears down (call
1098
+ // ended, user navigated away), without coupling that teardown to
1099
+ // the normal-effect re-run cycle above.
1100
+ useEffect(() => {
1101
+ return () => {
1102
+ try { voiceHandleRef.current.setHandsFree(false) } catch { /* ignore */ }
1103
+ try { setTurnLockRef.current('idle', 'cleanup') } catch { /* ignore */ }
1104
+ }
1105
+ }, [])
1106
+
1107
+ // Mirror the voice-controller's machine onto our UI state so the
1108
+ // halo + waveform + label actually reflect what's happening:
1109
+ // LISTENING β†’ 'listening'
1110
+ // THINKING β†’ 'thinking'
1111
+ // SPEAKING β†’ 'speaking'
1112
+ // IDLE β†’ 'listening' (ready, nothing to say yet)
1113
+ // OFF / muted β†’ preserved
1114
+ useEffect(() => {
1115
+ if (!connected) return
1116
+ if (muted) return
1117
+ const mapped = mapVoiceState(voice.state)
1118
+ if (!mapped) return
1119
+ setState(prev => {
1120
+ if (prev === 'ended' || prev === 'muted') return prev
1121
+ return mapped
1122
+ })
1123
+ }, [voice.state, connected, muted])
1124
+
1125
+ // Mirror the voice-controller's LISTENING / THINKING onto the
1126
+ // turn-lock 'user' slot. The AI owns 'ai' (see speakText); VAD
1127
+ // drives 'user' only when the AI isn't already speaking. This is
1128
+ // the industry-standard "who has the floor" gate β€” with it, the
1129
+ // AI's TTS bleed through the speaker can't trip a false user turn
1130
+ // (the root cause of the "AI says yes, hears itself, stops" bug).
1131
+ useEffect(() => {
1132
+ if (!fullDuplexEnabledRef.current) return
1133
+ if (!connected) return
1134
+ if (muted) return
1135
+ if (voice.state === 'LISTENING' && turnLockRef.current === 'idle') {
1136
+ clog({ e: 'vad', action: 'speech_start', state: voice.state })
1137
+ setTurnLock('user', 'voice_listening')
1138
+ } else if (voice.state === 'THINKING' && turnLockRef.current === 'user') {
1139
+ clog({ e: 'vad', action: 'speech_end', state: voice.state })
1140
+ setTurnLock('idle', 'voice_thinking')
1141
+ }
1142
+ }, [voice.state, connected, muted, setTurnLock])
1143
+
1144
+ // While the opener TTS is playing, the UI label must read
1145
+ // 'speaking' β€” the persona is literally talking. Default state
1146
+ // transitions would land on 'listening' (post-connect default)
1147
+ // which is a lie: audio is going OUT of the speaker, not coming
1148
+ // IN to the mic. The mic is still closed (see the openerPlaying
1149
+ // gate on setHandsFree above), so voice.state stays OFF during
1150
+ // this window and the mirror effect above doesn't override us.
1151
+ useEffect(() => {
1152
+ if (!openerPlaying) return
1153
+ setState(prev =>
1154
+ (prev === 'ended' || prev === 'muted') ? prev : 'speaking',
1155
+ )
1156
+ }, [openerPlaying])
1157
+
1158
+ // ── Live waveform intensity (0..1) ─────────────────────────────
1159
+ // One rAF loop per state-group writes to a ref that CallWaveform
1160
+ // reads directly (no React re-renders). React only sees the state
1161
+ // label; the bars themselves are driven straight from the DOM.
1162
+ //
1163
+ // listening / muted β†’ voice.audioLevel (real mic RMS)
1164
+ // speaking β†’ synthesised speech envelope β€” browser TTS
1165
+ // audio isn't routable through an AnalyserNode,
1166
+ // so we generate a natural-sounding envelope
1167
+ // (sum of 3 sines + a small noise floor). The
1168
+ // visual cadence matches syllable rate + the
1169
+ // amplitude range matches what mic analysers
1170
+ // produce on voiced speech, so the persona's
1171
+ // bars and the user's bars "look like the
1172
+ // same kind of thing."
1173
+ // thinking β†’ slow 0.15..0.25 drift β€” "processing"
1174
+ // dialing/connecting/ended β†’ 0.05 floor β€” visible but quiet
1175
+ const intensityRef = useRef(0.05)
1176
+ const voiceRef = useRef(voice)
1177
+ useEffect(() => { voiceRef.current = voice }, [voice])
1178
+
1179
+ useEffect(() => {
1180
+ let raf = 0
1181
+ const start = performance.now()
1182
+
1183
+ if (state === 'listening' || state === 'muted') {
1184
+ const tick = () => {
1185
+ // Real mic level. useVoiceController already smooths with
1186
+ // an EMA internally; we pass it through unmodified.
1187
+ intensityRef.current = muted ? 0 : voiceRef.current.audioLevel
1188
+ raf = requestAnimationFrame(tick)
1189
+ }
1190
+ raf = requestAnimationFrame(tick)
1191
+ return () => cancelAnimationFrame(raf)
1192
+ }
1193
+
1194
+ if (state === 'speaking') {
1195
+ const tick = () => {
1196
+ const t = (performance.now() - start) / 1000
1197
+ // Three sines at the frequencies you'd see in actual
1198
+ // voiced speech analysis: ~2 Hz syllable rate, ~5 Hz
1199
+ // mid-band, ~10 Hz high-band flutter. A small noise floor
1200
+ // breaks the periodicity so bars don't pulse in lockstep.
1201
+ const v =
1202
+ 0.45 +
1203
+ Math.sin(t * 2.3) * 0.22 +
1204
+ Math.sin(t * 5.7 + 1.3) * 0.16 +
1205
+ Math.sin(t * 11.1 + 2.7) * 0.09 +
1206
+ (Math.random() - 0.5) * 0.07
1207
+ intensityRef.current = Math.max(0, Math.min(1, v))
1208
+ raf = requestAnimationFrame(tick)
1209
+ }
1210
+ raf = requestAnimationFrame(tick)
1211
+ return () => cancelAnimationFrame(raf)
1212
+ }
1213
+
1214
+ if (state === 'thinking') {
1215
+ const tick = () => {
1216
+ const t = (performance.now() - start) / 1000
1217
+ intensityRef.current = 0.18 + Math.sin(t * 1.1) * 0.06
1218
+ raf = requestAnimationFrame(tick)
1219
+ }
1220
+ raf = requestAnimationFrame(tick)
1221
+ return () => cancelAnimationFrame(raf)
1222
+ }
1223
+
1224
+ // dialing / connecting / ended β†’ static low floor.
1225
+ intensityRef.current = 0.05
1226
+ }, [state, muted])
1227
+
1228
+ const toggleMute = useCallback(() => {
1229
+ setMuted((m) => {
1230
+ const next = !m
1231
+ if (next) {
1232
+ voice.setHandsFree(false)
1233
+ setState('muted')
1234
+ } else {
1235
+ voice.setHandsFree(true)
1236
+ setState('listening')
1237
+ }
1238
+ if (useBackendRef.current) {
1239
+ try { sessionRef.current.sendUiState({ muted: next }) } catch { /* ignore */ }
1240
+ }
1241
+ return next
1242
+ })
1243
+ }, [voice])
1244
+
1245
+ const handleMinimize = useCallback(() => {
1246
+ if (onMinimize) onMinimize()
1247
+ else onClose()
1248
+ }, [onMinimize, onClose])
1249
+
1250
+ return (
1251
+ <div
1252
+ role="dialog"
1253
+ aria-modal="true"
1254
+ aria-label={`Call with ${personaName}`}
1255
+ className="fixed inset-0 z-[100] flex items-center justify-center"
1256
+ >
1257
+ <div
1258
+ className="absolute inset-0"
1259
+ style={{
1260
+ background: HP_CALL.backdrop,
1261
+ backdropFilter: 'blur(20px)',
1262
+ WebkitBackdropFilter: 'blur(20px)',
1263
+ transition: 'opacity 200ms ease',
1264
+ opacity: state === 'ended' ? 0 : 1,
1265
+ }}
1266
+ />
1267
+ <div className="relative z-10">
1268
+ <CallModal
1269
+ state={state}
1270
+ personaName={personaName}
1271
+ imageUrl={avatarUrl}
1272
+ accentColor={accentColor}
1273
+ durationSec={durationSec}
1274
+ onEnd={handleEnd}
1275
+ onToggleMute={toggleMute}
1276
+ onMinimize={handleMinimize}
1277
+ onSwitchToChat={onSwitchToChat}
1278
+ intensityRef={intensityRef}
1279
+ />
1280
+ </div>
1281
+
1282
+ <style>{`
1283
+ @keyframes hp-call-in {
1284
+ from { opacity: 0; transform: scale(0.95); }
1285
+ to { opacity: 1; transform: scale(1); }
1286
+ }
1287
+ @keyframes hp-halo-breathe {
1288
+ 0%, 100% { opacity: 0.30; transform: scale(1); }
1289
+ 50% { opacity: 0.85; transform: scale(1.06); }
1290
+ }
1291
+ @keyframes hp-dot-pulse {
1292
+ 0%, 100% { transform: translateY(0); opacity: 0.35; }
1293
+ 50% { transform: translateY(-3px); opacity: 1; }
1294
+ }
1295
+ @keyframes hp-call-pulse-ring {
1296
+ 0% { transform: scale(1); opacity: 0.65; }
1297
+ 80% { opacity: 0; }
1298
+ 100% { transform: scale(1.55); opacity: 0; }
1299
+ }
1300
+ @keyframes hp-call-toast-in {
1301
+ from { opacity: 0; transform: translateY(8px); }
1302
+ to { opacity: 1; transform: translateY(0); }
1303
+ }
1304
+ `}</style>
1305
+ </div>
1306
+ )
1307
+ }
1308
+
1309
+
1310
+ // ── Call-ended toast β€” the "you just hung up, want to call back?" chip
1311
+
1312
+ export interface CallEndedToastProps {
1313
+ /** Duration of the live phase (seconds). Rendered as "42s" / "3m 12s". */
1314
+ durationSec: number
1315
+ onSpeakAgain: () => void
1316
+ onDismiss: () => void
1317
+ }
1318
+
1319
+ function formatEndedDuration(s: number): string {
1320
+ if (s < 60) return `${s}s`
1321
+ const m = Math.floor(s / 60)
1322
+ const rest = s % 60
1323
+ return rest === 0 ? `${m}m` : `${m}m ${rest}s`
1324
+ }
1325
+
1326
+ export const CallEndedToast: React.FC<CallEndedToastProps> = ({
1327
+ durationSec, onSpeakAgain, onDismiss,
1328
+ }) => {
1329
+ return (
1330
+ <div
1331
+ role="status"
1332
+ aria-live="polite"
1333
+ className="fixed bottom-6 left-1/2 z-[95] -translate-x-1/2"
1334
+ style={{
1335
+ animation: 'hp-call-toast-in 220ms ease-out',
1336
+ fontFamily: HP_CALL.font,
1337
+ }}
1338
+ >
1339
+ <div style={{
1340
+ display: 'flex', alignItems: 'center', gap: 12,
1341
+ padding: '8px 10px 8px 14px',
1342
+ borderRadius: 9999,
1343
+ background: HP_CALL.surface,
1344
+ border: `1px solid ${HP_CALL.border}`,
1345
+ boxShadow: '0 12px 32px rgba(0,0,0,0.45), inset 0 1px 0 rgba(255,255,255,0.04)',
1346
+ color: HP_CALL.text,
1347
+ }}>
1348
+ <span style={{
1349
+ display: 'inline-flex', alignItems: 'center', gap: 8,
1350
+ fontSize: 13, fontWeight: 500, color: HP_CALL.text2,
1351
+ }}>
1352
+ <span style={{
1353
+ width: 6, height: 6, borderRadius: '50%',
1354
+ background: HP_CALL.text3,
1355
+ }} />
1356
+ Call ended Β· <span style={{
1357
+ color: HP_CALL.text,
1358
+ fontFamily: HP_CALL.fontTabular,
1359
+ fontVariantNumeric: 'tabular-nums',
1360
+ }}>{formatEndedDuration(durationSec)}</span>
1361
+ </span>
1362
+ <button
1363
+ type="button"
1364
+ onClick={onSpeakAgain}
1365
+ style={{
1366
+ height: 30, padding: '0 14px', borderRadius: 9999,
1367
+ background: HP_CALL.accent,
1368
+ color: '#052c33', border: 'none', cursor: 'pointer',
1369
+ fontSize: 12, fontWeight: 600, letterSpacing: 0.2,
1370
+ }}
1371
+ >
1372
+ Speak again
1373
+ </button>
1374
+ <button
1375
+ type="button"
1376
+ onClick={onDismiss}
1377
+ aria-label="Dismiss"
1378
+ style={{
1379
+ width: 28, height: 28, borderRadius: 9999,
1380
+ background: 'transparent', border: 'none', cursor: 'pointer',
1381
+ color: HP_CALL.text3, fontSize: 18, lineHeight: 1, padding: 0,
1382
+ }}
1383
+ >
1384
+ Γ—
1385
+ </button>
1386
+ </div>
1387
+ </div>
1388
+ )
1389
+ }
frontend/src/ui/CreatorStudioEditor.tsx CHANGED
@@ -1847,6 +1847,12 @@ export function CreatorStudioEditor({
1847
  backendUrl={backendUrl}
1848
  videoId={projectId}
1849
  videoTitle={project?.title}
 
 
 
 
 
 
1850
  disabledReason={
1851
  scenes.length === 0
1852
  ? "Add at least one scene before exporting."
 
1847
  backendUrl={backendUrl}
1848
  videoId={projectId}
1849
  videoTitle={project?.title}
1850
+ scenes={scenes.map((s) => ({
1851
+ id: s.id,
1852
+ idx: s.idx,
1853
+ narration: s.narration || "",
1854
+ audioUrl: s.audioUrl || null,
1855
+ }))}
1856
  disabledReason={
1857
  scenes.length === 0
1858
  ? "Add at least one scene before exporting."
frontend/src/ui/SettingsPanel.tsx CHANGED
@@ -1,6 +1,15 @@
1
  import React, { useEffect, useMemo, useState } from "react";
2
  import OllamaHealthBanner from "./OllamaHealthBanner";
3
  import ProfileSettingsModal from "./ProfileSettingsModal";
 
 
 
 
 
 
 
 
 
4
  import { resolveBackendUrl } from "./lib/backendUrl";
5
  import {
6
  getModelSettings,
@@ -208,6 +217,12 @@ export default function SettingsPanel({
208
 
209
  // Voice state
210
  const [availableVoices, setAvailableVoices] = useState<SpeechSynthesisVoice[]>([]);
 
 
 
 
 
 
211
  const [voicesInitialized, setVoicesInitialized] = useState(false);
212
 
213
  // Video presets state (fetched from backend based on selected model)
@@ -1084,27 +1099,40 @@ export default function SettingsPanel({
1084
  <span className="text-xs text-white">Enable Text-to-Speech (TTS)</span>
1085
  </label>
1086
 
1087
- {/* Voice Selection */}
1088
- <div>
1089
- <label className="block text-[10px] text-white/50 mb-2">Assistant Voice</label>
1090
- <select
1091
- className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
1092
- value={value.selectedVoice ?? ''}
1093
- onChange={(e) => onChangeDraft({ ...value, selectedVoice: e.target.value })}
1094
- >
1095
- <option value="">System Default</option>
1096
- {availableVoices.map((voice) => (
1097
- <option key={voice.voiceURI} value={voice.name}>
1098
- {voice.name} ({voice.lang})
1099
- </option>
1100
- ))}
1101
- </select>
1102
- <div className="mt-1 text-[10px] text-white/40">
1103
- {availableVoices.length > 0
1104
- ? `Choose from ${availableVoices.length} available voices`
1105
- : 'Loading voices...'}
 
 
 
 
 
1106
  </div>
1107
- </div>
 
 
 
 
 
 
 
 
1108
  </div>
1109
  </div>
1110
 
 
1
  import React, { useEffect, useMemo, useState } from "react";
2
  import OllamaHealthBanner from "./OllamaHealthBanner";
3
  import ProfileSettingsModal from "./ProfileSettingsModal";
4
+ import TtsEngineSection from "./components/TtsEngineSection";
5
+ // Side-effect import: registers the bundled TTS providers (web-speech-api,
6
+ // piper-wasm). Importing here guarantees the registry is populated the
7
+ // first time the Settings panel mounts, before TtsEngineSection reads it.
8
+ import "./tts";
9
+ import {
10
+ getActiveTtsEngineId,
11
+ onActiveTtsEngineChange,
12
+ } from "./tts";
13
  import { resolveBackendUrl } from "./lib/backendUrl";
14
  import {
15
  getModelSettings,
 
217
 
218
  // Voice state
219
  const [availableVoices, setAvailableVoices] = useState<SpeechSynthesisVoice[]>([]);
220
+ // Track the active TTS engine so we can hide the legacy "Assistant
221
+ // Voice" select when the user picks a non-default engine (Piper etc.).
222
+ // Purely cosmetic: the underlying value.selectedVoice state is not
223
+ // touched β€” only the control rendering it.
224
+ const [ttsEngineId, setTtsEngineId] = useState<string>(() => getActiveTtsEngineId());
225
+ useEffect(() => onActiveTtsEngineChange(setTtsEngineId), []);
226
  const [voicesInitialized, setVoicesInitialized] = useState(false);
227
 
228
  // Video presets state (fetched from backend based on selected model)
 
1099
  <span className="text-xs text-white">Enable Text-to-Speech (TTS)</span>
1100
  </label>
1101
 
1102
+ {/* Voice Selection β€” only shown for the default engine. When
1103
+ the user picks a non-default engine (Piper etc.) the engine
1104
+ owns its own voice picker below, and showing two would be
1105
+ confusing duplication. */}
1106
+ {ttsEngineId === 'web-speech-api' && (
1107
+ <div>
1108
+ <label className="block text-[10px] text-white/50 mb-2">Assistant Voice</label>
1109
+ <select
1110
+ className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
1111
+ value={value.selectedVoice ?? ''}
1112
+ onChange={(e) => onChangeDraft({ ...value, selectedVoice: e.target.value })}
1113
+ >
1114
+ <option value="">System Default</option>
1115
+ {availableVoices.map((voice) => (
1116
+ <option key={voice.voiceURI} value={voice.name}>
1117
+ {voice.name} ({voice.lang})
1118
+ </option>
1119
+ ))}
1120
+ </select>
1121
+ <div className="mt-1 text-[10px] text-white/40">
1122
+ {availableVoices.length > 0
1123
+ ? `Choose from ${availableVoices.length} available voices`
1124
+ : 'Loading voices...'}
1125
+ </div>
1126
  </div>
1127
+ )}
1128
+
1129
+ {/* TTS Engine picker β€” additive plugin system. The existing
1130
+ System Voice selector above stays as the default path; this
1131
+ section lets the user opt into alternative engines (Piper
1132
+ WASM today; more can register themselves without touching
1133
+ this file). Schema-driven controls so future engines
1134
+ require zero Settings edits. */}
1135
+ <TtsEngineSection systemVoices={availableVoices} />
1136
  </div>
1137
  </div>
1138
 
frontend/src/ui/call/bargeIn.test.ts ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Unit tests for the pure detector in bargeIn.ts. Covers the exact
3
+ * Β§ 6 invariants (single-spike rejection, sustain gate, refractory).
4
+ * The hook form is intentionally not tested here β€” it's a thin
5
+ * wrapper around the detector + rAF; coverage lives in the overlay
6
+ * integration path.
7
+ */
8
+ import { describe, it, expect } from 'vitest'
9
+ import { detectBargeIn } from './bargeIn'
10
+
11
+ const OPTS = {
12
+ threshold: 0.08,
13
+ minSustainMs: 80,
14
+ minRefractoryMs: 400,
15
+ } as const
16
+
17
+ function genConstant(level: number, durMs: number, stepMs = 16) {
18
+ const out: Array<{ tMs: number; level: number }> = []
19
+ for (let t = 0; t <= durMs; t += stepMs) {
20
+ out.push({ tMs: t, level })
21
+ }
22
+ return out
23
+ }
24
+
25
+ describe('detectBargeIn', () => {
26
+ it('fires once after sustained speech above threshold', () => {
27
+ // Steady high level for 200 ms β€” comfortably over the 80 ms sustain.
28
+ const events = detectBargeIn(genConstant(0.2, 200), OPTS)
29
+ expect(events.length).toBe(1)
30
+ expect(events[0]).toBeGreaterThanOrEqual(OPTS.minSustainMs)
31
+ })
32
+
33
+ it('does NOT fire on a single spike below sustain', () => {
34
+ // One 16 ms spike followed by silence. Below 80 ms sustain β†’ drops.
35
+ const samples = [
36
+ { tMs: 0, level: 0.0 },
37
+ { tMs: 16, level: 0.2 },
38
+ { tMs: 32, level: 0.0 },
39
+ { tMs: 48, level: 0.0 },
40
+ { tMs: 64, level: 0.0 },
41
+ ]
42
+ expect(detectBargeIn(samples, OPTS)).toEqual([])
43
+ })
44
+
45
+ it('does NOT fire when level never exceeds threshold', () => {
46
+ const events = detectBargeIn(genConstant(0.04, 400), OPTS)
47
+ expect(events).toEqual([])
48
+ })
49
+
50
+ it('honours the refractory window (no double-fire during sustained speech)', () => {
51
+ // 1500 ms of steady speech. Without refractory we'd fire every
52
+ // tick after 80 ms. With refractory we fire at most once per
53
+ // minRefractoryMs (~400 ms).
54
+ const events = detectBargeIn(genConstant(0.2, 1500), OPTS)
55
+ expect(events.length).toBeGreaterThan(0)
56
+ // At 16 ms steps and 400 ms refractory: expect ~3 events.
57
+ expect(events.length).toBeLessThanOrEqual(4)
58
+ // Consecutive events must be at least refractory-ms apart.
59
+ for (let i = 1; i < events.length; i++) {
60
+ expect(events[i] - events[i - 1]).toBeGreaterThanOrEqual(
61
+ OPTS.minRefractoryMs,
62
+ )
63
+ }
64
+ })
65
+
66
+ it('resets the sustain counter after a dip below threshold', () => {
67
+ // 50 ms above, 30 ms below, 100 ms above. First run is too short;
68
+ // second run also too short on its own because counter reset.
69
+ const samples = [
70
+ ...Array.from({ length: 4 }, (_, i) => ({
71
+ tMs: i * 16, level: 0.2,
72
+ })),
73
+ { tMs: 64, level: 0.0 },
74
+ { tMs: 80, level: 0.0 },
75
+ ...Array.from({ length: 4 }, (_, i) => ({
76
+ tMs: 96 + i * 16, level: 0.2,
77
+ })),
78
+ ]
79
+ expect(detectBargeIn(samples, OPTS)).toEqual([])
80
+ })
81
+ })
frontend/src/ui/call/bargeIn.ts ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * bargeIn.ts β€” VAD-tap hook that detects user speech-start while
3
+ * the assistant's TTS is playing, so the client can interrupt the
4
+ * persona the way a person would.
5
+ *
6
+ * Design (Β§ 5.4 of docs/analysis/voice-call-streaming-design.md):
7
+ *
8
+ * - Reads the EMA-smoothed mic level that useVoiceController
9
+ * already exposes (audioLevelRef), sampled at rAF cadence.
10
+ * - Fires onBargeIn when the level stays above ``threshold`` for
11
+ * at least ``minSustainMs``. Sustain protects against a
12
+ * single spike (keyboard click, cough) cutting the persona off.
13
+ * - Only enabled when the overlay says it should be β€” i.e. the
14
+ * TTS is actually speaking AND the session negotiated streaming.
15
+ * Outside that window it's a pure no-op.
16
+ *
17
+ * The threshold here (default 0.08) is deliberately HIGHER than the
18
+ * baseline VAD threshold (~0.035) that useVoiceController uses for
19
+ * end-of-user-turn silence detection. Reason: during TTS playback a
20
+ * fraction of the persona's own audio returns into the mic on most
21
+ * laptops; a low threshold trips on that bleed and generates
22
+ * phantom barge-ins. 0.08 is above typical TTS bleed but well below
23
+ * normal voiced-speech peaks.
24
+ */
25
+
26
+ import { useEffect, useRef } from 'react'
27
+
28
+ export interface UseBargeInDetectorArgs {
29
+ /** Shared ref mutated at rAF cadence by the parent β€” typically
30
+ * ``voice.audioLevel`` mirrored into a ref so we avoid stale
31
+ * reads without thrashing React. 0..1 range. */
32
+ audioLevelRef: React.MutableRefObject<number>
33
+ /** Only fire barge-ins when this is true. Wire this to
34
+ * ``tts.isSpeaking && session.streamingNegotiated``. */
35
+ enabled: boolean
36
+ /** Callback fired once per speech-start episode. Debounced via
37
+ * ``minRefractoryMs`` to avoid double-firing on sustained speech. */
38
+ onBargeIn: () => void
39
+ /** Audio level (0..1) above which the tap considers the user to
40
+ * be speaking. Default 0.08 β€” see header comment on TTS bleed. */
41
+ threshold?: number
42
+ /** How long the level must stay above ``threshold`` before we
43
+ * declare speech-start. 80 ms matches the minimum voiced-phoneme
44
+ * duration, so a single keyboard clack doesn't trigger. */
45
+ minSustainMs?: number
46
+ /** After firing, ignore further triggers for this long. Prevents
47
+ * a single barge-in from firing twice while speech continues. */
48
+ minRefractoryMs?: number
49
+ }
50
+
51
+ /**
52
+ * Mount a VAD tap that fires ``onBargeIn`` when sustained speech is
53
+ * detected during an active TTS phase. Cleanup on unmount.
54
+ */
55
+ export function useBargeInDetector({
56
+ audioLevelRef,
57
+ enabled,
58
+ onBargeIn,
59
+ threshold = 0.08,
60
+ minSustainMs = 80,
61
+ minRefractoryMs = 400,
62
+ }: UseBargeInDetectorArgs): void {
63
+ // Latest callback in a ref so a change in ``onBargeIn`` identity
64
+ // doesn't tear down the whole rAF loop.
65
+ const onBargeInRef = useRef(onBargeIn)
66
+ useEffect(() => {
67
+ onBargeInRef.current = onBargeIn
68
+ }, [onBargeIn])
69
+
70
+ useEffect(() => {
71
+ if (!enabled) return
72
+ let raf = 0
73
+ let aboveSince: number | null = null
74
+ let refractoryUntil = 0
75
+ const tick = () => {
76
+ const now = performance.now()
77
+ const level = audioLevelRef.current ?? 0
78
+ if (level >= threshold) {
79
+ if (aboveSince === null) aboveSince = now
80
+ else if (
81
+ now >= refractoryUntil &&
82
+ now - aboveSince >= minSustainMs
83
+ ) {
84
+ refractoryUntil = now + minRefractoryMs
85
+ aboveSince = null
86
+ try {
87
+ onBargeInRef.current()
88
+ } catch {
89
+ /* listener errors must not crash the rAF loop */
90
+ }
91
+ }
92
+ } else {
93
+ aboveSince = null
94
+ }
95
+ raf = requestAnimationFrame(tick)
96
+ }
97
+ raf = requestAnimationFrame(tick)
98
+ return () => {
99
+ cancelAnimationFrame(raf)
100
+ }
101
+ }, [enabled, audioLevelRef, threshold, minSustainMs, minRefractoryMs])
102
+ }
103
+
104
+ // Exported for tests β€” pure detector without the React hook.
105
+ export function detectBargeIn(
106
+ samples: Array<{ tMs: number; level: number }>,
107
+ opts: {
108
+ threshold: number
109
+ minSustainMs: number
110
+ minRefractoryMs: number
111
+ },
112
+ ): number[] {
113
+ const events: number[] = []
114
+ let aboveSince: number | null = null
115
+ let refractoryUntil = 0
116
+ for (const { tMs, level } of samples) {
117
+ if (level >= opts.threshold) {
118
+ if (aboveSince === null) aboveSince = tMs
119
+ else if (
120
+ tMs >= refractoryUntil &&
121
+ tMs - aboveSince >= opts.minSustainMs
122
+ ) {
123
+ refractoryUntil = tMs + opts.minRefractoryMs
124
+ aboveSince = null
125
+ events.push(tMs)
126
+ }
127
+ } else {
128
+ aboveSince = null
129
+ }
130
+ }
131
+ return events
132
+ }
frontend/src/ui/call/callApi.ts ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * callApi.ts β€” REST wrapper for the voice_call session endpoint.
3
+ *
4
+ * Thin on purpose. Two responsibilities:
5
+ * 1. POST /v1/voice-call/sessions with the right shape
6
+ * 2. Normalize "feature unavailable" (404/501) vs hard errors
7
+ * so the hook can gracefully fall back to the chat-REST path
8
+ * when the backend has VOICE_CALL_ENABLED=false.
9
+ *
10
+ * Anything richer (envelope codec, seq bookkeeping, heartbeat,
11
+ * reconnect) lives in callSocket.ts β€” keeping those concerns out of
12
+ * this file is deliberate; the REST handshake runs once per call,
13
+ * the socket lifecycle runs for every envelope.
14
+ */
15
+
16
+ import { resolveBackendUrl } from '../lib/backendUrl'
17
+
18
+ export interface CreateCallSessionRequest {
19
+ /** Existing chat conversation id to link this call to. Optional.
20
+ * The backend will create an ephemeral conversation if omitted. */
21
+ conversation_id?: string | null
22
+ /** Persona to carry into the turn. Null = assistant default. */
23
+ persona_id?: string | null
24
+ /** How the call was initiated. Matches backend CreateSessionReq. */
25
+ entry_mode?: 'voice' | 'call' | 'handoff'
26
+ /** Free-form client fingerprint (tz, platform, ua family).
27
+ * Also carries Phase 2/3 capability declarations:
28
+ * - ``streaming`` client can consume assistant.partial
29
+ * - ``barge_in`` client can detect user speech-start during
30
+ * TTS playback and send ``user.barge_in``
31
+ * The effective streaming mode for the session is
32
+ * ``min(client_declared, server_advertised)`` β€” both ends must
33
+ * agree or the turn runs in unary mode. Mismatches degrade
34
+ * gracefully without a hard error. */
35
+ device_info?: Record<string, unknown>
36
+ }
37
+
38
+ /** Server-advertised capabilities. Fields are optional so older
39
+ * servers (which don't emit them) are parsed safely and treated
40
+ * as "no". Used by useCallSession to decide whether to subscribe
41
+ * to the Phase 2 streaming events. */
42
+ export interface CallCapabilities {
43
+ /** Server supports assistant.partial / assistant.turn_end /
44
+ * assistant.cancel envelopes. */
45
+ streaming?: boolean
46
+ /** Server honours user.barge_in / transcript.partial as cancel
47
+ * signals. Only meaningful when ``streaming`` is also true. */
48
+ barge_in?: boolean
49
+ /** Legacy fields (pre-streaming). Kept in the interface so
50
+ * existing consumers don't need to widen the type. */
51
+ interruptions?: boolean
52
+ transcript_live?: boolean
53
+ [k: string]: unknown
54
+ }
55
+
56
+ export interface CreateCallSessionResponse {
57
+ session_id: string
58
+ /** Absolute or root-relative WebSocket URL to connect to. */
59
+ ws_url: string
60
+ /** Short-lived token the WS handshake must echo in its query. */
61
+ resume_token: string
62
+ /** When the session expires if never connected (ISO or epoch ms). */
63
+ expires_at: string | number
64
+ /** Hard cap enforced by the server. */
65
+ max_duration_sec: number
66
+ /** Backend capability advertisement (Β§ 3.1 of the streaming design). */
67
+ capabilities?: CallCapabilities
68
+ }
69
+
70
+ /** Error raised for any non-2xx from /v1/voice-call/sessions. The
71
+ * `isUnavailable` flag distinguishes "backend hasn't enabled this
72
+ * feature yet" from real failures the UI should surface. */
73
+ export class CallApiError extends Error {
74
+ constructor(
75
+ public readonly status: number,
76
+ public readonly bodyText: string,
77
+ ) {
78
+ super(`voice-call session api ${status}${bodyText ? `: ${bodyText}` : ''}`)
79
+ this.name = 'CallApiError'
80
+ }
81
+ /** 404 = route not mounted (VOICE_CALL_ENABLED=false).
82
+ * 501 = feature recognized but disabled at runtime.
83
+ * The hook treats both as "fall back to chat REST". */
84
+ get isUnavailable(): boolean {
85
+ return this.status === 404 || this.status === 501
86
+ }
87
+ }
88
+
89
+ export async function createCallSession(
90
+ backendUrl: string,
91
+ req: CreateCallSessionRequest,
92
+ authToken?: string | null,
93
+ ): Promise<CreateCallSessionResponse> {
94
+ // Defensive resolution: when the caller passes an empty or
95
+ // whitespace-only backend URL (e.g. a stale settings load), a
96
+ // naive ``replace(/\/+$/, '')`` leaves us with ``''`` and fetch
97
+ // falls back to the current page origin. On the Vite dev server
98
+ // (:3000) that silently routes to the SPA, which returns
99
+ // index.html and the subsequent JSON parse fails; in prod it can
100
+ // hit the wrong origin entirely. ``resolveBackendUrl`` ignores
101
+ // empty overrides and falls through to the app-wide resolver
102
+ // (localStorage β†’ VITE_API_URL β†’ origin β†’ ``http://localhost:8000``),
103
+ // so we never accidentally issue a relative request.
104
+ const base = resolveBackendUrl(backendUrl)
105
+ const res = await fetch(`${base}/v1/voice-call/sessions`, {
106
+ method: 'POST',
107
+ headers: {
108
+ 'content-type': 'application/json',
109
+ ...(authToken ? { authorization: `Bearer ${authToken}` } : {}),
110
+ },
111
+ body: JSON.stringify({
112
+ conversation_id: req.conversation_id ?? null,
113
+ persona_id: req.persona_id ?? null,
114
+ entry_mode: req.entry_mode ?? 'call',
115
+ device_info: req.device_info ?? {},
116
+ }),
117
+ credentials: 'include',
118
+ })
119
+
120
+ if (!res.ok) {
121
+ const bodyText = await res.text().catch(() => '')
122
+ throw new CallApiError(res.status, bodyText)
123
+ }
124
+
125
+ const json = (await res.json()) as CreateCallSessionResponse
126
+ if (!json?.session_id || !json?.resume_token || !json?.ws_url) {
127
+ throw new CallApiError(res.status, 'malformed session payload')
128
+ }
129
+ return json
130
+ }
131
+
132
+ /** Declare the client's Phase 2/3 capabilities into the
133
+ * ``device_info`` blob. Set both to true in build configurations
134
+ * that include streamTts + bargeIn modules; set to false to run
135
+ * the session in forced-unary mode even against a streaming server
136
+ * (useful for bisecting regressions). */
137
+ export function clientStreamingCapabilities(
138
+ base: Record<string, unknown>,
139
+ enabled: boolean,
140
+ ): Record<string, unknown> {
141
+ if (!enabled) return base
142
+ return { ...base, streaming: true, barge_in: true }
143
+ }
144
+
145
+ /** Compute the effective streaming mode from the server's capability
146
+ * response and the client's declared support. Truth table in Β§ 3.1
147
+ * of docs/analysis/voice-call-streaming-design.md. */
148
+ export function effectiveStreamingMode(
149
+ serverCaps: CallCapabilities | undefined,
150
+ clientStreaming: boolean,
151
+ ): { streaming: boolean; barge_in: boolean } {
152
+ const streaming = !!(serverCaps?.streaming && clientStreaming)
153
+ const barge_in = streaming && !!serverCaps?.barge_in
154
+ return { streaming, barge_in }
155
+ }
156
+
157
+ /** Resolve a potentially-relative ws_url against the current page
158
+ * and the backend URL. Backends sometimes return `/v1/voice-call/ws`
159
+ * (path only) so the client can pick the right scheme + host. */
160
+ export function resolveWsUrl(
161
+ wsUrl: string,
162
+ sessionId: string,
163
+ resumeToken: string,
164
+ authToken: string | null,
165
+ backendUrl: string,
166
+ ): string {
167
+ let url: URL
168
+ if (/^wss?:\/\//i.test(wsUrl)) {
169
+ url = new URL(wsUrl)
170
+ } else if (wsUrl.startsWith('//')) {
171
+ url = new URL(`${location.protocol === 'https:' ? 'wss:' : 'ws:'}${wsUrl}`)
172
+ } else if (wsUrl.startsWith('/')) {
173
+ // Root-relative β€” derive scheme + host from backendUrl.
174
+ const backend = new URL(backendUrl)
175
+ const scheme = backend.protocol === 'https:' ? 'wss:' : 'ws:'
176
+ url = new URL(`${scheme}//${backend.host}${wsUrl}`)
177
+ } else {
178
+ url = new URL(wsUrl, backendUrl)
179
+ }
180
+
181
+ // Inject session_id if the server gave us a template (rare) or
182
+ // appended nothing; the standard shape is
183
+ // /v1/voice-call/ws/{session_id}.
184
+ if (!url.pathname.endsWith(sessionId)) {
185
+ url.pathname = url.pathname.replace(/\/+$/, '') + `/${sessionId}`
186
+ }
187
+ url.searchParams.set('resume_token', resumeToken)
188
+ if (authToken) url.searchParams.set('token', authToken)
189
+ return url.toString()
190
+ }
frontend/src/ui/call/callSocket.ts ADDED
@@ -0,0 +1,579 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * callSocket.ts β€” typed transport for the voice_call WebSocket.
3
+ *
4
+ * Owns the connection, the envelope codec, monotonic sequence
5
+ * tracking, the heartbeat, graceful shutdown, and reconnect-with-
6
+ * backoff semantics. Exposes a narrow event surface that the React
7
+ * hook (useCallSession) binds to; callers never touch the raw socket.
8
+ *
9
+ * Explicit state machine β€” the only transitions allowed are:
10
+ *
11
+ * idle ──connect()──▢ connecting ──open───────▢ live
12
+ * β”‚ β”‚
13
+ * β”‚ (handshake failure) β”‚ (transient drop w/ resume window)
14
+ * β–Ό β–Ό
15
+ * closed reconnecting ──(give up / expired)──▢ closed
16
+ * β”‚
17
+ * └─────(socket open)───────▢ live
18
+ *
19
+ * live ──close('user_ended' | 'max_duration' | 'idle')───▢ closed
20
+ *
21
+ * Industry choices baked in:
22
+ * β€’ Exponential backoff with full jitter (AWS / Cloudflare canonical
23
+ * implementation) β€” stops thundering herds on a shared-backend
24
+ * brown-out.
25
+ * β€’ Heartbeat treats silence as liveness failure β€” we send a
26
+ * client-originated `call.control ping` every N seconds AND
27
+ * watch for server-side `ping` events; if BOTH are silent for
28
+ * 2Γ— the interval, the connection is considered dead and the
29
+ * backoff loop fires.
30
+ * β€’ Outbound envelopes are queued while reconnecting. When the
31
+ * socket reopens within the resume window the queue is flushed
32
+ * in-order; otherwise it's dropped on shutdown.
33
+ * β€’ Resume semantics honoured: one reconnect attempt carries the
34
+ * same session_id + resume_token; if the server returns
35
+ * 1008 resume-expired we give up and surface 'resume_expired'.
36
+ * β€’ Every event emitted by this module is typed; no `any`, no
37
+ * ad-hoc payload parsing inside listeners.
38
+ */
39
+
40
+ // ── Typed envelope contracts ───────────────────────────────────────
41
+ // Mirror the backend ws.py contract. Keep in sync with
42
+ // docs/analysis/voice-call-human-simulation-design.md Β§ 6.
43
+
44
+ export type CallLifecycleStatus =
45
+ | 'idle'
46
+ | 'connecting'
47
+ | 'live'
48
+ | 'reconnecting'
49
+ | 'draining'
50
+ | 'closed'
51
+
52
+ export type CallCloseReason =
53
+ | 'user_ended'
54
+ | 'max_duration'
55
+ | 'idle_timeout'
56
+ | 'resume_expired'
57
+ | 'bad_resume_token'
58
+ | 'session_not_found'
59
+ | 'session_ended'
60
+ | 'websocket_disabled'
61
+ | 'network_error'
62
+ | 'unmounted'
63
+ | 'unknown'
64
+
65
+ export interface ServerEnvelope<
66
+ T extends string = string,
67
+ P extends Record<string, unknown> = Record<string, unknown>,
68
+ > {
69
+ type: T
70
+ seq: number
71
+ ts: number
72
+ payload: P
73
+ }
74
+
75
+ // Server β†’ client event payloads. One type per server `type`; if
76
+ // the backend ever adds a new event, it lands in `unknown` and is
77
+ // dropped β€” not a runtime crash.
78
+
79
+ export interface CallStatePayload {
80
+ status: 'live' | 'ending' | 'ended'
81
+ reason?: 'user_ended' | 'idle' | 'max_duration'
82
+ since?: number
83
+ }
84
+ export interface AssistantTranscriptPayload {
85
+ role: 'assistant'
86
+ text: string
87
+ }
88
+ export interface AssistantFillerPayload {
89
+ token: string
90
+ ts?: number
91
+ session_id?: string
92
+ }
93
+ export interface AssistantBackchannelPayload {
94
+ token: string
95
+ volume_db?: number
96
+ ts?: number
97
+ }
98
+ export interface ServerErrorPayload {
99
+ code: string
100
+ message: string
101
+ }
102
+
103
+ // Phase 2/3 payloads (Β§ 3.2 of the streaming design doc).
104
+
105
+ export interface AssistantPartialPayload {
106
+ turn_id: string
107
+ /** The NEW text only. Callers concatenate deltas per turn_id. */
108
+ delta: string
109
+ /** Monotonic per-turn index starting at 0. Advisory (the WS
110
+ * already enforces monotonic seq); useful for replay logs. */
111
+ index: number
112
+ }
113
+ export interface AssistantTurnEndPayload {
114
+ turn_id: string
115
+ reason: 'complete' | 'cancelled' | 'error'
116
+ /** Authoritative concatenation of this turn's deltas (server-side).
117
+ * Clients can use this directly instead of maintaining their own
118
+ * buffer. Optional in principle β€” but our server always emits it. */
119
+ full_text?: string
120
+ }
121
+ export interface AssistantCancelPayload {
122
+ turn_id: string
123
+ cause?: 'user_barge_in' | 'user_partial' | string
124
+ }
125
+
126
+ // Client β†’ server. Keep the types literal so the compiler catches
127
+ // typos at the send() call site.
128
+
129
+ export interface UiStatePayload {
130
+ muted?: boolean
131
+ speaker_on?: boolean
132
+ backgrounded?: boolean
133
+ }
134
+ export interface TranscriptFinalPayload {
135
+ text: string
136
+ model?: string
137
+ lang?: string
138
+ }
139
+ export type CallControlAction = 'end' | 'ping'
140
+
141
+ // ── Typed event bus for callers ────────────────────────────────────
142
+
143
+ export interface CallSocketEventMap {
144
+ statusChange: CallLifecycleStatus
145
+ callState: CallStatePayload
146
+ /** Still fires for turns that ran in unary mode (flag off OR
147
+ * capability mismatch). Streaming turns use the three events
148
+ * below instead. */
149
+ assistantTranscript: AssistantTranscriptPayload
150
+ /** Phase 2: one delta per server emission. */
151
+ assistantPartial: AssistantPartialPayload
152
+ /** Phase 2: exactly once per streamed turn, with a final reason. */
153
+ assistantTurnEnd: AssistantTurnEndPayload
154
+ /** Phase 3: server acks a barge-in; client stops TTS on receipt. */
155
+ assistantCancel: AssistantCancelPayload
156
+ assistantFiller: AssistantFillerPayload
157
+ assistantBackchannel: AssistantBackchannelPayload
158
+ serverError: ServerErrorPayload
159
+ safetyNotice: Record<string, unknown>
160
+ pong: void
161
+ closed: { reason: CallCloseReason; code?: number; detail?: string }
162
+ }
163
+
164
+ type Listener<E extends keyof CallSocketEventMap> = (
165
+ payload: CallSocketEventMap[E],
166
+ ) => void
167
+
168
+ // ── Backoff helper (AWS-style full jitter) ─────────────────────────
169
+
170
+ function backoffMs(attempt: number, baseMs = 500, capMs = 10_000): number {
171
+ const exp = Math.min(capMs, baseMs * 2 ** attempt)
172
+ return Math.floor(Math.random() * exp)
173
+ }
174
+
175
+ // ── Close-code β†’ reason mapping ────────────────────────────────────
176
+
177
+ const POLICY_VIOLATION = 1008
178
+ const NORMAL_CLOSURE = 1000
179
+
180
+ function closeReasonFrom(code: number | undefined, text: string): CallCloseReason {
181
+ const reason = (text || '').toLowerCase()
182
+ if (code === POLICY_VIOLATION) {
183
+ if (reason.includes('resume-expired')) return 'resume_expired'
184
+ if (reason.includes('bad-resume-token')) return 'bad_resume_token'
185
+ if (reason.includes('session-not-found')) return 'session_not_found'
186
+ if (reason.includes('session-ended')) return 'session_ended'
187
+ if (reason.includes('websocket-disabled')) return 'websocket_disabled'
188
+ }
189
+ if (code === NORMAL_CLOSURE && reason.includes('max-duration')) return 'max_duration'
190
+ if (reason.includes('idle')) return 'idle_timeout'
191
+ return 'unknown'
192
+ }
193
+
194
+ // ── Construction options ───────────────────────────────────────────
195
+
196
+ export interface CallSocketOptions {
197
+ /** Fully-resolved WS URL including ?resume_token=… and ?token=…. */
198
+ url: string
199
+ /** How long the backend will honour a resume. Used to decide when to
200
+ * give up reconnecting. Default 20 s (matches VOICE_CALL_RESUME_WINDOW_SEC). */
201
+ resumeWindowSec?: number
202
+ /** How often we send a client-originated ping. Default 15 s. */
203
+ heartbeatIntervalMs?: number
204
+ /** Dependency injection for tests β€” defaults to global WebSocket. */
205
+ webSocketImpl?: typeof WebSocket
206
+ /** Structured log sink. Defaults to console.{info,warn,error} with
207
+ * any `resume_token`, `token`, or `jwt` fields redacted by the
208
+ * caller. Keep string-typed; don't pass raw envelopes in. */
209
+ log?: {
210
+ info: (msg: string, extra?: Record<string, unknown>) => void
211
+ warn: (msg: string, extra?: Record<string, unknown>) => void
212
+ error: (msg: string, extra?: Record<string, unknown>) => void
213
+ }
214
+ }
215
+
216
+ // ── CallSocket class ───────────────────────────────────────────────
217
+
218
+ export class CallSocket {
219
+ private ws: WebSocket | null = null
220
+ private status: CallLifecycleStatus = 'idle'
221
+ private lastServerSeq = 0
222
+ private outboundQueue: string[] = []
223
+ private heartbeatTimer: number | null = null
224
+ private lastServerActivityMs = 0
225
+ private deadPeerTimer: number | null = null
226
+ private reconnectAttempt = 0
227
+ private reconnectTimer: number | null = null
228
+ private connectedAtMs = 0
229
+ private shuttingDown = false
230
+
231
+ private readonly listeners: {
232
+ [K in keyof CallSocketEventMap]?: Set<Listener<K>>
233
+ } = {}
234
+
235
+ private readonly url: string
236
+ private readonly resumeWindowMs: number
237
+ private readonly heartbeatIntervalMs: number
238
+ private readonly WebSocketImpl: typeof WebSocket
239
+ private readonly log: NonNullable<CallSocketOptions['log']>
240
+
241
+ constructor(opts: CallSocketOptions) {
242
+ this.url = opts.url
243
+ this.resumeWindowMs = (opts.resumeWindowSec ?? 20) * 1000
244
+ this.heartbeatIntervalMs = opts.heartbeatIntervalMs ?? 15_000
245
+ this.WebSocketImpl = opts.webSocketImpl ?? WebSocket
246
+ this.log = opts.log ?? {
247
+ info: (m, e) => console.info(`[callSocket] ${m}`, e ?? ''),
248
+ warn: (m, e) => console.warn(`[callSocket] ${m}`, e ?? ''),
249
+ error: (m, e) => console.error(`[callSocket] ${m}`, e ?? ''),
250
+ }
251
+ }
252
+
253
+ // Public API ----------------------------------------------------
254
+
255
+ getStatus(): CallLifecycleStatus { return this.status }
256
+
257
+ on<E extends keyof CallSocketEventMap>(evt: E, fn: Listener<E>): () => void {
258
+ let set = this.listeners[evt] as Set<Listener<E>> | undefined
259
+ if (!set) {
260
+ set = new Set<Listener<E>>()
261
+ // Narrow through `as unknown` β€” TS can't track the generic
262
+ // discriminant through the indexed property set.
263
+ ;(this.listeners as Record<string, unknown>)[evt] = set
264
+ }
265
+ set.add(fn)
266
+ return () => set!.delete(fn)
267
+ }
268
+
269
+ /** Open the socket. Safe to call exactly once per instance. */
270
+ connect(): void {
271
+ if (this.status !== 'idle') {
272
+ this.log.warn('connect() called in non-idle state', { status: this.status })
273
+ return
274
+ }
275
+ this.openSocket()
276
+ }
277
+
278
+ /** Send the captured user transcript as a chat turn. Queued if the
279
+ * socket is currently mid-reconnect. */
280
+ sendTranscript(p: TranscriptFinalPayload): void {
281
+ this.enqueueLine({ type: 'transcript.final', ts: Date.now(), payload: p })
282
+ }
283
+
284
+ /** Bookkeeping β€” muted, speaker on/off, app backgrounded. */
285
+ sendUiState(p: UiStatePayload): void {
286
+ this.enqueueLine({ type: 'ui.state', ts: Date.now(), payload: p })
287
+ }
288
+
289
+ /** Control channel β€” 'ping' requests a pong; 'end' terminates. */
290
+ sendControl(action: CallControlAction): void {
291
+ this.enqueueLine({ type: 'call.control', ts: Date.now(), payload: { action } })
292
+ }
293
+
294
+ /** Phase 2 β€” interim STT output while the user is still speaking.
295
+ * Server uses it as a secondary barge-in trigger and, in a
296
+ * future phase, for semantic endpointing. Safe no-op against a
297
+ * non-streaming server (unknown type is dropped). */
298
+ sendTranscriptPartial(p: { text: string; stable_prefix_len?: number }): void {
299
+ this.enqueueLine({
300
+ type: 'transcript.partial',
301
+ ts: Date.now(),
302
+ payload: p,
303
+ })
304
+ }
305
+
306
+ /** Phase 3 β€” explicit interrupt. Fired the instant the client's
307
+ * VAD trips above threshold while an assistant turn is in-flight.
308
+ * ``turn_id`` disambiguates against a stale signal racing a new
309
+ * turn; the server compares ids and silently drops mismatches. */
310
+ sendBargeIn(turn_id: string): void {
311
+ this.enqueueLine({
312
+ type: 'user.barge_in',
313
+ ts: Date.now(),
314
+ payload: { turn_id },
315
+ })
316
+ }
317
+
318
+ /** Graceful user-initiated end. Sends control 'end', then closes. */
319
+ end(): void {
320
+ if (this.shuttingDown) return
321
+ this.shuttingDown = true
322
+ this.transition('draining')
323
+ this.sendControl('end')
324
+ // Give the server ~400 ms to echo call.state {status:'ended'}
325
+ // before we force-close; prevents the socket from looking
326
+ // orphaned in backend logs.
327
+ window.setTimeout(() => this.dispose('user_ended'), 400)
328
+ }
329
+
330
+ /** Teardown without a control 'end' β€” used when the component
331
+ * unmounts, the user navigates away, or a terminal error occurs. */
332
+ dispose(reason: CallCloseReason = 'unmounted', code = NORMAL_CLOSURE): void {
333
+ if (this.status === 'closed') return
334
+ this.shuttingDown = true
335
+ this.clearTimers()
336
+ try { this.ws?.close(code, reason) } catch { /* already closed */ }
337
+ this.ws = null
338
+ this.outboundQueue = []
339
+ this.transition('closed')
340
+ this.emit('closed', { reason, code })
341
+ }
342
+
343
+ // Socket lifecycle ----------------------------------------------
344
+
345
+ private openSocket(): void {
346
+ this.transition(this.reconnectAttempt === 0 ? 'connecting' : 'reconnecting')
347
+ let ws: WebSocket
348
+ try {
349
+ ws = new this.WebSocketImpl(this.url)
350
+ } catch (err) {
351
+ this.log.error('WebSocket constructor threw', { err: String(err) })
352
+ this.scheduleReconnect()
353
+ return
354
+ }
355
+ this.ws = ws
356
+ this.lastServerActivityMs = Date.now()
357
+
358
+ ws.addEventListener('open', () => {
359
+ this.reconnectAttempt = 0
360
+ this.connectedAtMs = Date.now()
361
+ this.transition('live')
362
+ this.startHeartbeat()
363
+ this.flushQueue()
364
+ })
365
+
366
+ ws.addEventListener('message', (ev) => this.onMessage(ev))
367
+
368
+ ws.addEventListener('close', (ev) => this.onClose(ev))
369
+
370
+ ws.addEventListener('error', () => {
371
+ // The browser fires `error` right before `close` on network
372
+ // failures. We don't transition here β€” onClose() decides
373
+ // reconnect vs terminal based on the code.
374
+ this.log.warn('socket error event')
375
+ })
376
+ }
377
+
378
+ private onMessage(ev: MessageEvent): void {
379
+ this.lastServerActivityMs = Date.now()
380
+ if (typeof ev.data !== 'string') return
381
+
382
+ let env: ServerEnvelope | null = null
383
+ try {
384
+ env = JSON.parse(ev.data) as ServerEnvelope
385
+ } catch {
386
+ this.log.warn('non-JSON frame dropped')
387
+ return
388
+ }
389
+ if (!env || typeof env.type !== 'string') return
390
+
391
+ // Monotonic seq validation. If the server ever ships out-of-order
392
+ // frames we log once and keep going β€” correctness trumps strictness.
393
+ if (typeof env.seq === 'number' && env.seq <= this.lastServerSeq) {
394
+ this.log.warn('non-monotonic seq', {
395
+ got: env.seq, last: this.lastServerSeq, type: env.type,
396
+ })
397
+ } else if (typeof env.seq === 'number') {
398
+ this.lastServerSeq = env.seq
399
+ }
400
+
401
+ this.dispatch(env)
402
+ }
403
+
404
+ private dispatch(env: ServerEnvelope): void {
405
+ const raw = env.payload as unknown
406
+ switch (env.type) {
407
+ case 'call.state': {
408
+ const p = raw as CallStatePayload
409
+ this.emit('callState', p)
410
+ if (p.status === 'ended') this.dispose('user_ended')
411
+ return
412
+ }
413
+ case 'transcript.final':
414
+ this.emit('assistantTranscript', raw as AssistantTranscriptPayload)
415
+ return
416
+ case 'assistant.partial':
417
+ this.emit('assistantPartial', raw as AssistantPartialPayload)
418
+ return
419
+ case 'assistant.turn_end':
420
+ this.emit('assistantTurnEnd', raw as AssistantTurnEndPayload)
421
+ return
422
+ case 'assistant.cancel':
423
+ this.emit('assistantCancel', raw as AssistantCancelPayload)
424
+ return
425
+ case 'assistant.filler':
426
+ this.emit('assistantFiller', raw as AssistantFillerPayload)
427
+ return
428
+ case 'assistant.backchannel':
429
+ this.emit('assistantBackchannel', raw as AssistantBackchannelPayload)
430
+ return
431
+ case 'safety.notice':
432
+ this.emit('safetyNotice', env.payload)
433
+ return
434
+ case 'error':
435
+ this.emit('serverError', raw as ServerErrorPayload)
436
+ return
437
+ case 'pong':
438
+ this.emit('pong', undefined as void)
439
+ return
440
+ case 'ping':
441
+ // Server heartbeat β€” immediately echo to keep the liveness
442
+ // counters tight on both ends.
443
+ this.sendControl('ping')
444
+ return
445
+ default:
446
+ // Forward-compat: unknown event types are no-ops.
447
+ this.log.info(`unknown server event: ${env.type}`)
448
+ }
449
+ }
450
+
451
+ private onClose(ev: CloseEvent): void {
452
+ const reason = closeReasonFrom(ev.code, ev.reason)
453
+ this.clearTimers()
454
+
455
+ // Terminal close codes: don't reconnect.
456
+ const terminal: ReadonlySet<CallCloseReason> = new Set([
457
+ 'user_ended',
458
+ 'max_duration',
459
+ 'resume_expired',
460
+ 'bad_resume_token',
461
+ 'session_not_found',
462
+ 'session_ended',
463
+ 'websocket_disabled',
464
+ ])
465
+ if (this.shuttingDown || terminal.has(reason)) {
466
+ this.ws = null
467
+ this.transition('closed')
468
+ this.emit('closed', { reason, code: ev.code, detail: ev.reason })
469
+ return
470
+ }
471
+
472
+ // Transient β€” try to reconnect within the resume window.
473
+ this.log.warn('socket dropped; scheduling reconnect', {
474
+ code: ev.code, reason: ev.reason,
475
+ })
476
+ this.ws = null
477
+ this.scheduleReconnect()
478
+ }
479
+
480
+ private scheduleReconnect(): void {
481
+ if (this.shuttingDown) return
482
+ const sinceLiveMs = this.connectedAtMs > 0 ? Date.now() - this.connectedAtMs : 0
483
+ if (sinceLiveMs > this.resumeWindowMs) {
484
+ this.log.warn('resume window elapsed; giving up')
485
+ this.dispose('resume_expired', POLICY_VIOLATION)
486
+ return
487
+ }
488
+ const delay = backoffMs(this.reconnectAttempt)
489
+ this.reconnectAttempt += 1
490
+ this.transition('reconnecting')
491
+ this.reconnectTimer = window.setTimeout(() => {
492
+ this.reconnectTimer = null
493
+ this.openSocket()
494
+ }, delay)
495
+ }
496
+
497
+ // Queue + heartbeat ---------------------------------------------
498
+
499
+ private enqueueLine(env: { type: string; ts: number; payload: unknown }): void {
500
+ const line = JSON.stringify(env)
501
+ if (this.status === 'live' && this.ws?.readyState === this.WebSocketImpl.OPEN) {
502
+ try { this.ws.send(line) } catch (err) {
503
+ this.log.warn('send failed; queueing', { err: String(err) })
504
+ this.outboundQueue.push(line)
505
+ }
506
+ } else {
507
+ this.outboundQueue.push(line)
508
+ }
509
+ }
510
+
511
+ private flushQueue(): void {
512
+ if (!this.ws || this.ws.readyState !== this.WebSocketImpl.OPEN) return
513
+ const q = this.outboundQueue
514
+ this.outboundQueue = []
515
+ for (const line of q) {
516
+ try { this.ws.send(line) } catch (err) {
517
+ this.log.error('flush failed; re-queueing remaining', { err: String(err) })
518
+ this.outboundQueue.push(line)
519
+ return
520
+ }
521
+ }
522
+ }
523
+
524
+ private startHeartbeat(): void {
525
+ this.clearHeartbeat()
526
+ this.heartbeatTimer = window.setInterval(() => {
527
+ this.sendControl('ping')
528
+ }, this.heartbeatIntervalMs)
529
+
530
+ // Dead-peer watchdog β€” if we haven't heard from the server in
531
+ // 2Γ— the heartbeat interval, force a reconnect. Avoids hanging
532
+ // forever on half-open TCP sockets (mobile NAT rebinds).
533
+ this.deadPeerTimer = window.setInterval(() => {
534
+ if (Date.now() - this.lastServerActivityMs > this.heartbeatIntervalMs * 2) {
535
+ this.log.warn('peer silent; forcing reconnect')
536
+ try { this.ws?.close() } catch { /* ignore */ }
537
+ }
538
+ }, this.heartbeatIntervalMs)
539
+ }
540
+
541
+ private clearHeartbeat(): void {
542
+ if (this.heartbeatTimer !== null) {
543
+ window.clearInterval(this.heartbeatTimer)
544
+ this.heartbeatTimer = null
545
+ }
546
+ if (this.deadPeerTimer !== null) {
547
+ window.clearInterval(this.deadPeerTimer)
548
+ this.deadPeerTimer = null
549
+ }
550
+ }
551
+
552
+ private clearTimers(): void {
553
+ this.clearHeartbeat()
554
+ if (this.reconnectTimer !== null) {
555
+ window.clearTimeout(this.reconnectTimer)
556
+ this.reconnectTimer = null
557
+ }
558
+ }
559
+
560
+ // Emit + state transition ---------------------------------------
561
+
562
+ private emit<E extends keyof CallSocketEventMap>(
563
+ evt: E, payload: CallSocketEventMap[E],
564
+ ): void {
565
+ const set = this.listeners[evt] as Set<Listener<E>> | undefined
566
+ if (!set) return
567
+ for (const fn of set) {
568
+ try { fn(payload) } catch (err) {
569
+ this.log.error('listener threw', { evt, err: String(err) })
570
+ }
571
+ }
572
+ }
573
+
574
+ private transition(next: CallLifecycleStatus): void {
575
+ if (this.status === next) return
576
+ this.status = next
577
+ this.emit('statusChange', next)
578
+ }
579
+ }
frontend/src/ui/call/log.ts ADDED
@@ -0,0 +1,140 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Structured call-lifecycle logger + shared TTS ownership.
3
+ *
4
+ * Three pieces, shipped together because they're tightly coupled:
5
+ *
6
+ * clog(evt) Namespaced, grep-friendly console log
7
+ * for every call-related event. Format
8
+ * ``[Call+<ms>] {e:…, …}`` so
9
+ * ``grep "\\[Call\\]"`` surfaces a
10
+ * chronological trace of any session.
11
+ *
12
+ * speakOwned(source, text) Single-owner TTS dispatch. Both the
13
+ * overlay's fallback opener and the
14
+ * App-level assistant-reply effect route
15
+ * through this function, which guards
16
+ * on ``window.__hp_tts_owner__`` so
17
+ * duplicate sources don't double-speak
18
+ * the same turn. First to call wins;
19
+ * the second sees ``owner !== source``
20
+ * and drops cleanly (emits an
21
+ * ``tts: dedupe`` log so the drop is
22
+ * auditable).
23
+ *
24
+ * isCallFullDuplexEnabled() Feature flag gate. Defaults to ON; a
25
+ * user can force-off via
26
+ * ``localStorage.homepilot_call_full_duplex``
27
+ * or a build-time ``VITE_CALL_FULL_DUPLEX_ENABLED``.
28
+ * With the flag off every consumer keeps
29
+ * its pre-flag behaviour β€” lets us flip
30
+ * the full-duplex fix on/off during
31
+ * dogfood without editing code.
32
+ *
33
+ * No runtime DOM setup; safe to import from SSR contexts β€” the
34
+ * speakOwned path guards on ``typeof window``.
35
+ */
36
+
37
+ export type CallLogEvent =
38
+ | { e: 'lifecycle'; phase: 'open' | 'ready' | 'greet' | 'hangup' | 'close'; sid?: string }
39
+ | { e: 'tts'; action: 'start' | 'end' | 'dedupe'; source: 'overlay' | 'app-level'; textLen: number }
40
+ | { e: 'vad'; action: 'speech_start' | 'speech_end' | 'suppressed'; level?: number; state: string }
41
+ | { e: 'turn'; action: 'user_out' | 'assistant_in' | 'barge_in'; route: 'ws' | 'chat_rest'; latencyMs?: number }
42
+ | { e: 'state'; from: string; to: string; trigger: string }
43
+ | { e: 'error'; where: string; message: string }
44
+
45
+ declare global {
46
+ interface Window {
47
+ __hp_tts_owner__?: 'overlay' | 'app-level' | null
48
+ }
49
+ }
50
+
51
+ export function clog(evt: CallLogEvent): void {
52
+ if (typeof performance === 'undefined') return
53
+ const ts = Math.floor(performance.now())
54
+ // console.info so DevTools 'Info' filter can hide it in prod if
55
+ // needed; keeps the log present for debug without noise.
56
+ // eslint-disable-next-line no-console
57
+ console.info(`[Call+${ts}]`, evt)
58
+ }
59
+
60
+ /** Resolve the feature-flag state. Priority:
61
+ * 1. ``localStorage.homepilot_call_full_duplex`` override
62
+ * 2. build-time ``VITE_CALL_FULL_DUPLEX_ENABLED`` (default 'true')
63
+ * Falls through to ``true`` on any read error. */
64
+ export function isCallFullDuplexEnabled(): boolean {
65
+ if (typeof window === 'undefined') return true
66
+ try {
67
+ const override = window.localStorage.getItem('homepilot_call_full_duplex')
68
+ if (override === 'true') return true
69
+ if (override === 'false') return false
70
+ } catch {
71
+ /* ignore storage errors */
72
+ }
73
+ const envVal = (import.meta as unknown as {
74
+ env?: Record<string, string | undefined>
75
+ }).env?.VITE_CALL_FULL_DUPLEX_ENABLED
76
+ return String(envVal ?? 'true') !== 'false'
77
+ }
78
+
79
+ export interface SpeakOwnedHooks {
80
+ onStart?: () => void
81
+ onEnd?: () => void
82
+ onError?: (message: string) => void
83
+ }
84
+
85
+ /**
86
+ * Speak ``text`` via ``window.SpeechService`` with a shared owner
87
+ * lock. Returns ``true`` if this call owns the utterance, ``false``
88
+ * if another owner was already speaking (dedupe). Hooks fire on
89
+ * the owning path only.
90
+ *
91
+ * Fallback when the shim isn't available: emits the ``tts.start``
92
+ * log + an estimated ``tts.end`` timer anyway, so callers still
93
+ * get their ``onEnd`` hook (used by the mic-gate release in
94
+ * ``CallOverlay``) even with no TTS engine present.
95
+ */
96
+ export function speakOwned(
97
+ source: 'overlay' | 'app-level',
98
+ text: string,
99
+ hooks?: SpeakOwnedHooks,
100
+ ): boolean {
101
+ if (typeof window === 'undefined') return false
102
+ const t = (text ?? '').trim()
103
+ if (!t) return false
104
+ const owner = window.__hp_tts_owner__
105
+ if (owner && owner !== source) {
106
+ clog({ e: 'tts', action: 'dedupe', source, textLen: t.length })
107
+ return false
108
+ }
109
+ window.__hp_tts_owner__ = source
110
+ try {
111
+ hooks?.onStart?.()
112
+ clog({ e: 'tts', action: 'start', source, textLen: t.length })
113
+ const svc = (window as unknown as {
114
+ SpeechService?: { speak?: (text: string) => void }
115
+ }).SpeechService
116
+ svc?.speak?.(t)
117
+ // Length-estimate for the onEnd release β€” the SpeechService shim
118
+ // in this repo doesn't expose an explicit onend hook, so we
119
+ // guesstimate from text length + cap it so a runaway payload
120
+ // can't permanently hold the owner lock. ~12 chars/s typical;
121
+ // [800 ms, 10 s] clamp.
122
+ const estimateMs = Math.max(800, Math.min(10_000, t.length * 80))
123
+ window.setTimeout(() => {
124
+ if (window.__hp_tts_owner__ === source) {
125
+ window.__hp_tts_owner__ = null
126
+ }
127
+ hooks?.onEnd?.()
128
+ clog({ e: 'tts', action: 'end', source, textLen: t.length })
129
+ }, estimateMs)
130
+ return true
131
+ } catch (err) {
132
+ if (window.__hp_tts_owner__ === source) {
133
+ window.__hp_tts_owner__ = null
134
+ }
135
+ const message = err instanceof Error ? err.message : String(err)
136
+ hooks?.onError?.(message)
137
+ clog({ e: 'error', where: 'speakOwned', message })
138
+ return false
139
+ }
140
+ }
frontend/src/ui/call/streamTts.test.ts ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Unit tests for streamTts β€” the pluggable streaming TTS surface.
3
+ * Anchored to Β§ 5.3 of docs/analysis/voice-call-streaming-design.md.
4
+ */
5
+ import { describe, it, expect } from 'vitest'
6
+ import { createStreamingTts, streamTtsInternals } from './streamTts'
7
+
8
+ const { splitIntoSentences, NullStreamTts } = streamTtsInternals
9
+
10
+ describe('splitIntoSentences', () => {
11
+ it('splits on sentence enders followed by whitespace', () => {
12
+ const { sentences, remainder } = splitIntoSentences(
13
+ 'hello there. how are you? fine! and you',
14
+ )
15
+ expect(sentences).toEqual([
16
+ 'hello there.',
17
+ 'how are you?',
18
+ 'fine!',
19
+ ])
20
+ expect(remainder).toBe('and you')
21
+ })
22
+
23
+ it('keeps partial trailing text as remainder', () => {
24
+ const { sentences, remainder } = splitIntoSentences('partial sentence')
25
+ expect(sentences).toEqual([])
26
+ expect(remainder).toBe('partial sentence')
27
+ })
28
+
29
+ it('is safe on empty input', () => {
30
+ const { sentences, remainder } = splitIntoSentences('')
31
+ expect(sentences).toEqual([])
32
+ expect(remainder).toBe('')
33
+ })
34
+
35
+ it('splits on newlines + semicolons', () => {
36
+ const { sentences, remainder } = splitIntoSentences(
37
+ 'one thing;\n another thing.\n tail',
38
+ )
39
+ // Exact count + order matters so regressions are obvious.
40
+ expect(sentences.length).toBeGreaterThanOrEqual(2)
41
+ expect(sentences[0]).toContain('one thing')
42
+ expect(remainder.trim()).toBe('tail')
43
+ })
44
+ })
45
+
46
+ describe('NullStreamTts fallback', () => {
47
+ it('is safe to call every method; never claims to be speaking', () => {
48
+ const tts = new NullStreamTts()
49
+ expect(tts.isSpeaking).toBe(false)
50
+ tts.appendDelta('hello')
51
+ tts.flush()
52
+ tts.stop()
53
+ expect(tts.isSpeaking).toBe(false)
54
+ })
55
+ })
56
+
57
+ describe('createStreamingTts factory', () => {
58
+ it('returns an object implementing the StreamingTts contract', () => {
59
+ const tts = createStreamingTts()
60
+ expect(typeof tts.appendDelta).toBe('function')
61
+ expect(typeof tts.flush).toBe('function')
62
+ expect(typeof tts.stop).toBe('function')
63
+ expect(typeof tts.isSpeaking).toBe('boolean')
64
+ })
65
+ })
frontend/src/ui/call/streamTts.ts ADDED
@@ -0,0 +1,173 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * streamTts.ts β€” pluggable streaming text-to-speech surface.
3
+ *
4
+ * One interface, three planned implementations. This file ships the
5
+ * first (browser Web Speech); Piper-WASM and remote vendor streamers
6
+ * are follow-ups that just swap the concrete constructor in
7
+ * :func:`createStreamingTts`.
8
+ *
9
+ * Contract (Β§ 5.3 of docs/analysis/voice-call-streaming-design.md):
10
+ *
11
+ * appendDelta(delta) push a new text chunk. Implementations
12
+ * flush to the audio engine at sentence
13
+ * boundaries; callers hand over raw deltas
14
+ * without worrying about clause shaping.
15
+ * flush() signal end-of-turn. Any residual buffer
16
+ * is spoken; the engine transitions back
17
+ * to idle once the audio drains.
18
+ * stop() barge-in. MUST silence any audio playing
19
+ * or scheduled to play within ≀ 50 ms on
20
+ * Chromium. Cheap to call redundantly.
21
+ * isSpeaking read-only; true while audio is playing or
22
+ * scheduled. Drives the VAD-tap enable gate.
23
+ */
24
+
25
+ export interface StreamingTts {
26
+ appendDelta(delta: string): void
27
+ flush(): void
28
+ stop(): void
29
+ readonly isSpeaking: boolean
30
+ }
31
+
32
+ // ── Sentence-boundary helper ──────────────────────────────────────────
33
+ //
34
+ // Flush on: . ! ? ; or a newline β€” but only when they're followed
35
+ // by whitespace or end-of-buffer (so "e.g." / "Dr." don't mis-split).
36
+ // A tiny regex, kept readable rather than clever; the failure mode on
37
+ // a miss is "one extra utterance per turn," not a correctness bug.
38
+
39
+ const SENTENCE_END = /([.!?;\n])\s+/g
40
+
41
+ function splitIntoSentences(buffer: string): {
42
+ sentences: string[]
43
+ remainder: string
44
+ } {
45
+ const sentences: string[] = []
46
+ let lastIndex = 0
47
+ let m: RegExpExecArray | null
48
+ SENTENCE_END.lastIndex = 0
49
+ while ((m = SENTENCE_END.exec(buffer)) !== null) {
50
+ const end = m.index + m[0].length
51
+ const piece = buffer.slice(lastIndex, end).trim()
52
+ if (piece) sentences.push(piece)
53
+ lastIndex = end
54
+ }
55
+ const remainder = buffer.slice(lastIndex)
56
+ return { sentences, remainder }
57
+ }
58
+
59
+ // ── Web Speech implementation (ships first) ──────────────────────────
60
+ //
61
+ // Uses ``window.speechSynthesis`` directly. Each sentence becomes
62
+ // exactly one SpeechSynthesisUtterance so barge-in (via ``cancel()``)
63
+ // drops the current sentence + the queue atomically.
64
+ //
65
+ // Why one utterance per sentence instead of one per delta:
66
+ //
67
+ // 1. SpeechSynthesis queues utterances, not characters. Passing every
68
+ // delta through would create 40+ queued utterances per turn on
69
+ // Chromium, each with its own intonation reset β€” sounds robotic.
70
+ // 2. Cancel semantics are per-utterance, so sentence-granularity
71
+ // means barge-in can interrupt mid-sentence without stranding
72
+ // half-spoken buffers.
73
+
74
+ class WebSpeechStreamTts implements StreamingTts {
75
+ private buffer = ''
76
+ private speakingCount = 0
77
+ private stopped = false
78
+ private readonly syn: SpeechSynthesis | null
79
+
80
+ constructor() {
81
+ this.syn =
82
+ typeof window !== 'undefined' && 'speechSynthesis' in window
83
+ ? window.speechSynthesis
84
+ : null
85
+ }
86
+
87
+ get isSpeaking(): boolean {
88
+ return this.speakingCount > 0
89
+ }
90
+
91
+ appendDelta(delta: string): void {
92
+ if (this.stopped || !delta) return
93
+ this.buffer += delta
94
+ const { sentences, remainder } = splitIntoSentences(this.buffer)
95
+ this.buffer = remainder
96
+ for (const s of sentences) this._speak(s)
97
+ }
98
+
99
+ flush(): void {
100
+ if (this.stopped) return
101
+ if (this.buffer.trim()) {
102
+ this._speak(this.buffer.trim())
103
+ this.buffer = ''
104
+ }
105
+ }
106
+
107
+ stop(): void {
108
+ // Idempotent β€” safe to call repeatedly from the barge-in path.
109
+ this.stopped = true
110
+ this.buffer = ''
111
+ this.speakingCount = 0
112
+ try {
113
+ this.syn?.cancel()
114
+ } catch {
115
+ /* cancel is best-effort */
116
+ }
117
+ }
118
+
119
+ private _speak(text: string): void {
120
+ if (!this.syn) return
121
+ let utter: SpeechSynthesisUtterance
122
+ try {
123
+ utter = new SpeechSynthesisUtterance(text)
124
+ } catch {
125
+ return
126
+ }
127
+ // Track speaking state so the VAD-tap enable gate stays tight
128
+ // even when the queue reorders utterances (Safari / old Chromium).
129
+ this.speakingCount += 1
130
+ const done = () => {
131
+ this.speakingCount = Math.max(0, this.speakingCount - 1)
132
+ }
133
+ utter.onend = done
134
+ utter.onerror = done
135
+ try {
136
+ this.syn.speak(utter)
137
+ } catch {
138
+ done()
139
+ }
140
+ }
141
+ }
142
+
143
+ // ── Null implementation β€” used in tests + as a safety fallback ───────
144
+
145
+ class NullStreamTts implements StreamingTts {
146
+ isSpeaking = false
147
+ appendDelta(_delta: string): void { /* no-op */ }
148
+ flush(): void { /* no-op */ }
149
+ stop(): void { /* no-op */ }
150
+ }
151
+
152
+ // ── Factory ───────────────────────────────────────────────────────────
153
+ //
154
+ // Returns the best implementation available at runtime. Swapping
155
+ // in Piper-WASM or a remote vendor streamer is a one-line change
156
+ // in this function β€” call sites don't need to know.
157
+
158
+ export function createStreamingTts(): StreamingTts {
159
+ if (typeof window === 'undefined') return new NullStreamTts()
160
+ if (!('speechSynthesis' in window)) return new NullStreamTts()
161
+ // Probe for the existence of an app-level SpeechService shim; if
162
+ // present we could route through it, but for now the direct Web
163
+ // Speech path is both simpler and lower-latency.
164
+ return new WebSpeechStreamTts()
165
+ }
166
+
167
+ // Exported for tests + rare callers that want an explicit unplugged
168
+ // implementation (e.g. when a user opts out of TTS entirely).
169
+ export const streamTtsInternals = {
170
+ splitIntoSentences,
171
+ WebSpeechStreamTts,
172
+ NullStreamTts,
173
+ }
frontend/src/ui/call/useCallSession.ts ADDED
@@ -0,0 +1,381 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * useCallSession β€” React hook that owns the voice_call lifecycle.
3
+ *
4
+ * One instance per mounted `CallOverlayInner`. Composes callApi +
5
+ * callSocket into a single, cleanup-safe React surface:
6
+ *
7
+ * const session = useCallSession({ enabled, backendUrl, authToken,
8
+ * conversationId, personaId,
9
+ * deviceInfo })
10
+ *
11
+ * session.status β†’ 'idle' | 'creating' | 'connecting'
12
+ * | 'live' | 'reconnecting'
13
+ * | 'unavailable' | 'error' | 'closed'
14
+ * session.sendTranscript(text) β†’ route STT capture to the backend
15
+ * session.end() β†’ graceful user-initiated close
16
+ * session.onAssistantTranscript β†’ subscribe to assistant replies
17
+ * session.onAssistantFiller β†’ subscribe to "hmm…" events
18
+ * session.onAssistantBackchannel β†’ subscribe to "mm-hm" events
19
+ * session.lastError β†’ surface for the UI
20
+ *
21
+ * When `enabled` flips to true we:
22
+ * 1. POST /v1/voice-call/sessions (via callApi)
23
+ * 2. Open the WebSocket (via callSocket)
24
+ * 3. Wire status + event forwarding into React state
25
+ *
26
+ * Critically, if the POST returns 404/501 (backend has
27
+ * VOICE_CALL_ENABLED=false) we surface `status='unavailable'` and
28
+ * stop β€” the caller then falls back to the chat-REST path. That
29
+ * graceful-degradation path is what lets us ship this module
30
+ * independent of the backend flag rollout.
31
+ */
32
+
33
+ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
34
+ import {
35
+ createCallSession,
36
+ resolveWsUrl,
37
+ CallApiError,
38
+ type CreateCallSessionRequest,
39
+ type CreateCallSessionResponse,
40
+ } from './callApi'
41
+ import {
42
+ CallSocket,
43
+ type AssistantBackchannelPayload,
44
+ type AssistantCancelPayload,
45
+ type AssistantFillerPayload,
46
+ type AssistantPartialPayload,
47
+ type AssistantTranscriptPayload,
48
+ type AssistantTurnEndPayload,
49
+ type CallCloseReason,
50
+ type CallLifecycleStatus,
51
+ } from './callSocket'
52
+
53
+ export type CallSessionStatus =
54
+ | 'idle'
55
+ | 'creating'
56
+ | 'connecting'
57
+ | 'live'
58
+ | 'reconnecting'
59
+ | 'draining'
60
+ | 'closed'
61
+ | 'unavailable'
62
+ | 'error'
63
+
64
+ export interface UseCallSessionArgs {
65
+ /** The hook is inert until this flips to true. Flipping back to
66
+ * false triggers a graceful dispose. */
67
+ enabled: boolean
68
+ /** Resolved HomePilot backend URL (scheme + host). */
69
+ backendUrl: string
70
+ /** Bearer JWT for the REST handshake + WS query param. */
71
+ authToken: string | null
72
+ /** Session-creation request fields. */
73
+ request: CreateCallSessionRequest
74
+ }
75
+
76
+ export interface CallSessionHandle {
77
+ status: CallSessionStatus
78
+ /** Server-side canonical status last seen (live / ending / ended). */
79
+ callState: 'live' | 'ending' | 'ended' | null
80
+ /** Raw close reason once the session terminates. Useful for
81
+ * differentiating user-end vs timeout vs backend-disabled. */
82
+ closeReason: CallCloseReason | null
83
+ /** Any terminal error. 'unavailable' is NOT an error β€” it's a
84
+ * legitimate graceful-degradation signal. */
85
+ lastError: string | null
86
+ /** Route a final STT transcript into the call turn. No-op outside
87
+ * the 'live' | 'reconnecting' states. */
88
+ sendTranscript: (text: string) => void
89
+ /** Graceful end β€” sends call.control end, closes the socket. */
90
+ end: () => void
91
+ /** Bookkeeping. Server persists the *type* of the event only. */
92
+ sendUiState: (p: { muted?: boolean; speaker_on?: boolean; backgrounded?: boolean }) => void
93
+ /** Phase 2 β€” interim user STT (secondary barge-in signal). */
94
+ sendTranscriptPartial: (p: { text: string; stable_prefix_len?: number }) => void
95
+ /** Phase 3 β€” explicit user interrupt carrying the active turn_id. */
96
+ sendBargeIn: (turn_id: string) => void
97
+ /** Whether this session negotiated streaming. Pulled from the
98
+ * session-create capabilities response. False = unary mode. */
99
+ streamingNegotiated: boolean
100
+ /** Whether this session negotiated barge-in (implies streaming). */
101
+ bargeInNegotiated: boolean
102
+ /** Subscribe helpers β€” each returns an unsubscribe fn. Stable
103
+ * identities via useRef-backed implementation so the caller can
104
+ * pass them to useEffect deps without re-subscription storms. */
105
+ onAssistantTranscript: (fn: (p: AssistantTranscriptPayload) => void) => () => void
106
+ onAssistantFiller: (fn: (p: AssistantFillerPayload) => void) => () => void
107
+ onAssistantBackchannel: (fn: (p: AssistantBackchannelPayload) => void) => () => void
108
+ /** Phase 2 subscribes. */
109
+ onAssistantPartial: (fn: (p: AssistantPartialPayload) => void) => () => void
110
+ onAssistantTurnEnd: (fn: (p: AssistantTurnEndPayload) => void) => () => void
111
+ onAssistantCancel: (fn: (p: AssistantCancelPayload) => void) => () => void
112
+ }
113
+
114
+ // When the backend returns 404/501 for voice_call (VOICE_CALL_ENABLED=false)
115
+ // we remember the miss per-backend and short-circuit future session creates
116
+ // for UNAVAILABLE_BACKOFF_MS. Without this, every CallOverlay mount posts a
117
+ // fresh 404 against the same flag, spamming the console and delaying the
118
+ // fallback path by a full network round-trip.
119
+ const UNAVAILABLE_BACKOFF_MS = 30_000
120
+ const unavailableUntilByBackend = new Map<string, number>()
121
+
122
+ export function useCallSession(args: UseCallSessionArgs): CallSessionHandle {
123
+ const { enabled, backendUrl, authToken, request } = args
124
+
125
+ const [status, setStatus] = useState<CallSessionStatus>('idle')
126
+ const [callState, setCallState] = useState<'live' | 'ending' | 'ended' | null>(null)
127
+ const [closeReason, setCloseReason] = useState<CallCloseReason | null>(null)
128
+ const [lastError, setLastError] = useState<string | null>(null)
129
+
130
+ const socketRef = useRef<CallSocket | null>(null)
131
+ const sessionRef = useRef<CreateCallSessionResponse | null>(null)
132
+
133
+ // Stable subscribe surface. Keep listener sets here so they survive
134
+ // socket reconnects β€” the old socket's listeners are torn down with
135
+ // it, but user subscriptions persist across reconnects and are
136
+ // re-attached when a new socket comes up.
137
+ const txListeners = useRef(new Set<(p: AssistantTranscriptPayload) => void>())
138
+ const fillerListeners = useRef(new Set<(p: AssistantFillerPayload) => void>())
139
+ const bcListeners = useRef(new Set<(p: AssistantBackchannelPayload) => void>())
140
+ // Phase 2/3 listener sets. Kept separate so an old subscribe set
141
+ // doesn't churn when a new one mounts (stream wire-up rebinds often
142
+ // during CallOverlay renders).
143
+ const partialListeners = useRef(new Set<(p: AssistantPartialPayload) => void>())
144
+ const turnEndListeners = useRef(new Set<(p: AssistantTurnEndPayload) => void>())
145
+ const cancelListeners = useRef(new Set<(p: AssistantCancelPayload) => void>())
146
+ // Negotiated modes β€” re-derived on session create, stashed here so
147
+ // every render reads the same value without re-running the effect.
148
+ const [streamingNegotiated, setStreamingNegotiated] = useState(false)
149
+ const [bargeInNegotiated, setBargeInNegotiated] = useState(false)
150
+
151
+ // Ref'd primitive so the session-create effect doesn't re-trigger
152
+ // when the request object identity changes every render.
153
+ const requestRef = useRef(request)
154
+ useEffect(() => { requestRef.current = request }, [request])
155
+
156
+ // Main lifecycle. Runs once per (enabled, backendUrl, authToken)
157
+ // triple β€” which is what we want: flipping enabled true/false
158
+ // drives open/close; swapping backendUrl or re-auth reconnects.
159
+ useEffect(() => {
160
+ if (!enabled) return
161
+
162
+ let disposed = false
163
+ const teardown = () => {
164
+ disposed = true
165
+ socketRef.current?.dispose('unmounted')
166
+ socketRef.current = null
167
+ sessionRef.current = null
168
+ }
169
+
170
+ const run = async () => {
171
+ setStatus('creating')
172
+ setLastError(null)
173
+ setCloseReason(null)
174
+ setCallState(null)
175
+
176
+ // Skip the POST entirely if this backend has been 404/501-ing
177
+ // recently. Saves a round-trip per call re-entry while the flag
178
+ // is off server-side; the backoff window expires automatically
179
+ // when we clear the map entry below on a success.
180
+ const skipUntil = unavailableUntilByBackend.get(backendUrl)
181
+ if (skipUntil && Date.now() < skipUntil) {
182
+ // eslint-disable-next-line no-console
183
+ console.info(
184
+ '[useCallSession] skipping createCallSession β€” backend flagged unavailable',
185
+ { backendUrl, resumesAt: new Date(skipUntil).toISOString() },
186
+ )
187
+ setStatus('unavailable')
188
+ return
189
+ }
190
+
191
+ let handshake: CreateCallSessionResponse
192
+ try {
193
+ handshake = await createCallSession(backendUrl, requestRef.current, authToken)
194
+ } catch (err) {
195
+ if (disposed) return
196
+ if (err instanceof CallApiError && err.isUnavailable) {
197
+ unavailableUntilByBackend.set(
198
+ backendUrl,
199
+ Date.now() + UNAVAILABLE_BACKOFF_MS,
200
+ )
201
+ // eslint-disable-next-line no-console
202
+ console.warn(
203
+ '[useCallSession] voice_call unavailable β€” falling back to chat REST for',
204
+ `${UNAVAILABLE_BACKOFF_MS}ms`,
205
+ )
206
+ setStatus('unavailable')
207
+ return
208
+ }
209
+ // eslint-disable-next-line no-console
210
+ console.error('[useCallSession] createCallSession failed', err)
211
+ setStatus('error')
212
+ setLastError(err instanceof Error ? err.message : String(err))
213
+ return
214
+ }
215
+ if (disposed) return
216
+
217
+ // Handshake worked β€” clear any prior backoff so new flag flips
218
+ // take effect immediately instead of waiting out the window.
219
+ unavailableUntilByBackend.delete(backendUrl)
220
+
221
+ sessionRef.current = handshake
222
+ const url = resolveWsUrl(
223
+ handshake.ws_url,
224
+ handshake.session_id,
225
+ handshake.resume_token,
226
+ authToken,
227
+ backendUrl,
228
+ )
229
+
230
+ const sock = new CallSocket({ url })
231
+ socketRef.current = sock
232
+
233
+ // Translate socket lifecycle β†’ hook status.
234
+ sock.on('statusChange', (s: CallLifecycleStatus) => {
235
+ if (disposed) return
236
+ setStatus(s === 'idle' ? 'idle' : s)
237
+ })
238
+ sock.on('callState', (p) => {
239
+ if (disposed) return
240
+ setCallState(p.status)
241
+ })
242
+ sock.on('closed', ({ reason }) => {
243
+ if (disposed) return
244
+ setStatus('closed')
245
+ setCloseReason(reason)
246
+ })
247
+ sock.on('serverError', (p) => {
248
+ // Non-terminal by contract; surface for the UI but keep the
249
+ // socket open. The server will close explicitly if fatal.
250
+ if (disposed) return
251
+ setLastError(`${p.code}: ${p.message}`)
252
+ })
253
+ sock.on('assistantTranscript', (p) => {
254
+ for (const fn of txListeners.current) fn(p)
255
+ })
256
+ sock.on('assistantFiller', (p) => {
257
+ for (const fn of fillerListeners.current) fn(p)
258
+ })
259
+ sock.on('assistantBackchannel', (p) => {
260
+ for (const fn of bcListeners.current) fn(p)
261
+ })
262
+ // Phase 2/3 event fan-outs.
263
+ sock.on('assistantPartial', (p) => {
264
+ for (const fn of partialListeners.current) fn(p)
265
+ })
266
+ sock.on('assistantTurnEnd', (p) => {
267
+ for (const fn of turnEndListeners.current) fn(p)
268
+ })
269
+ sock.on('assistantCancel', (p) => {
270
+ for (const fn of cancelListeners.current) fn(p)
271
+ })
272
+
273
+ // Reflect negotiated modes from the session-create response.
274
+ const caps = handshake.capabilities as
275
+ | { streaming?: boolean; barge_in?: boolean } | undefined
276
+ setStreamingNegotiated(!!caps?.streaming)
277
+ setBargeInNegotiated(!!caps?.streaming && !!caps?.barge_in)
278
+
279
+ sock.connect()
280
+ }
281
+
282
+ void run()
283
+ return teardown
284
+ }, [enabled, backendUrl, authToken])
285
+
286
+ // Public actions. Guarded against stale-socket calls. --------------
287
+
288
+ const sendTranscript = useCallback((text: string) => {
289
+ const sock = socketRef.current
290
+ if (!sock) return
291
+ const trimmed = text.trim()
292
+ if (!trimmed) return
293
+ sock.sendTranscript({ text: trimmed })
294
+ }, [])
295
+
296
+ const end = useCallback(() => {
297
+ socketRef.current?.end()
298
+ }, [])
299
+
300
+ const sendUiState = useCallback(
301
+ (p: { muted?: boolean; speaker_on?: boolean; backgrounded?: boolean }) => {
302
+ socketRef.current?.sendUiState(p)
303
+ },
304
+ [],
305
+ )
306
+
307
+ const sendTranscriptPartial = useCallback(
308
+ (p: { text: string; stable_prefix_len?: number }) => {
309
+ socketRef.current?.sendTranscriptPartial(p)
310
+ },
311
+ [],
312
+ )
313
+
314
+ const sendBargeIn = useCallback((turn_id: string) => {
315
+ socketRef.current?.sendBargeIn(turn_id)
316
+ }, [])
317
+
318
+ const onAssistantTranscript = useCallback(
319
+ (fn: (p: AssistantTranscriptPayload) => void) => {
320
+ txListeners.current.add(fn)
321
+ return () => { txListeners.current.delete(fn) }
322
+ }, [],
323
+ )
324
+ const onAssistantFiller = useCallback(
325
+ (fn: (p: AssistantFillerPayload) => void) => {
326
+ fillerListeners.current.add(fn)
327
+ return () => { fillerListeners.current.delete(fn) }
328
+ }, [],
329
+ )
330
+ const onAssistantBackchannel = useCallback(
331
+ (fn: (p: AssistantBackchannelPayload) => void) => {
332
+ bcListeners.current.add(fn)
333
+ return () => { bcListeners.current.delete(fn) }
334
+ }, [],
335
+ )
336
+ const onAssistantPartial = useCallback(
337
+ (fn: (p: AssistantPartialPayload) => void) => {
338
+ partialListeners.current.add(fn)
339
+ return () => { partialListeners.current.delete(fn) }
340
+ }, [],
341
+ )
342
+ const onAssistantTurnEnd = useCallback(
343
+ (fn: (p: AssistantTurnEndPayload) => void) => {
344
+ turnEndListeners.current.add(fn)
345
+ return () => { turnEndListeners.current.delete(fn) }
346
+ }, [],
347
+ )
348
+ const onAssistantCancel = useCallback(
349
+ (fn: (p: AssistantCancelPayload) => void) => {
350
+ cancelListeners.current.add(fn)
351
+ return () => { cancelListeners.current.delete(fn) }
352
+ }, [],
353
+ )
354
+
355
+ return useMemo<CallSessionHandle>(() => ({
356
+ status,
357
+ callState,
358
+ closeReason,
359
+ lastError,
360
+ sendTranscript,
361
+ end,
362
+ sendUiState,
363
+ sendTranscriptPartial,
364
+ sendBargeIn,
365
+ streamingNegotiated,
366
+ bargeInNegotiated,
367
+ onAssistantTranscript,
368
+ onAssistantFiller,
369
+ onAssistantBackchannel,
370
+ onAssistantPartial,
371
+ onAssistantTurnEnd,
372
+ onAssistantCancel,
373
+ }), [
374
+ status, callState, closeReason, lastError,
375
+ sendTranscript, end, sendUiState,
376
+ sendTranscriptPartial, sendBargeIn,
377
+ streamingNegotiated, bargeInNegotiated,
378
+ onAssistantTranscript, onAssistantFiller, onAssistantBackchannel,
379
+ onAssistantPartial, onAssistantTurnEnd, onAssistantCancel,
380
+ ])
381
+ }
frontend/src/ui/components/CreatorExportWizard.tsx CHANGED
@@ -1,28 +1,65 @@
1
  /**
2
- * CreatorExportWizard β€” 2-step modal for exporting a Creator Studio video.
 
3
  *
4
  * Flow:
5
- * 1. Choose target β€” "MP4 (edit/share)" or "MP4 for YouTube".
6
- * 2. Render + download β€” POST /studio/videos/{id}/export/mp4, poll the
7
- * job, then trigger a same-tab download via <a href download>.
 
 
 
 
8
  *
9
  * Race protection:
10
- * The poller captures the wizard's mount-instance id at submit time and
11
- * discards any response that returns after the wizard has been closed
12
- * or after the user submitted a different render. This mirrors the
13
- * session-epoch pattern used in App.tsx for account-switch races.
14
  */
15
  import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
16
- import { Download, Film, Youtube, X, CheckCircle2, AlertTriangle, Loader2 } from 'lucide-react'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
 
18
  export type ExportKind = 'mp4_plain' | 'mp4_youtube'
19
 
 
 
 
 
 
 
 
 
 
20
  interface Props {
21
  open: boolean
22
  onClose: () => void
23
  backendUrl: string
24
  videoId: string
25
- /** Display name shown in the modal header (project / video title). */
 
 
26
  videoTitle?: string
27
  /** Disable submit when there are no scenes or scenes are still generating. */
28
  disabledReason?: string | null
@@ -40,6 +77,7 @@ interface JobView {
40
 
41
  const POLL_INTERVAL_MS = 1000
42
  const POLL_TIMEOUT_MS = 10 * 60 * 1000 // 10 minutes
 
43
 
44
  function authHeaders(): Record<string, string> {
45
  const token = localStorage.getItem('homepilot_auth_token') || ''
@@ -50,35 +88,74 @@ function trimBackend(u: string): string {
50
  return u.replace(/\/+$/, '')
51
  }
52
 
 
 
53
  export default function CreatorExportWizard({
54
  open,
55
  onClose,
56
  backendUrl,
57
  videoId,
 
58
  videoTitle,
59
  disabledReason,
60
  }: Props) {
61
- const [step, setStep] = useState<1 | 2>(1)
 
 
 
62
  const [kind, setKind] = useState<ExportKind>('mp4_youtube')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  const [job, setJob] = useState<JobView | null>(null)
64
  const [submitError, setSubmitError] = useState<string | null>(null)
65
 
66
- // Race protection: a monotonically-increasing token; every submit captures
67
- // the current value, and the poller drops responses whose epoch < current.
68
  const epochRef = useRef(0)
69
 
70
- // Reset modal whenever it opens.
71
  useEffect(() => {
72
  if (open) {
73
  setStep(1)
74
  setJob(null)
75
  setSubmitError(null)
 
 
 
 
 
76
  } else {
77
- // Closing the modal invalidates any in-flight poll.
78
  epochRef.current += 1
 
79
  }
80
  }, [open])
81
 
 
 
 
 
 
82
  const downloadUrl = useMemo(() => {
83
  if (!job || job.status !== 'done') return ''
84
  const tok = localStorage.getItem('homepilot_auth_token') || ''
@@ -86,12 +163,115 @@ export default function CreatorExportWizard({
86
  return `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${job.id}/download${qp}`
87
  }, [job, backendUrl, videoId])
88
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
89
  const pollJob = useCallback(
90
  async (jobId: string, myEpoch: number) => {
91
  const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${jobId}`
92
  const start = Date.now()
93
- // Polling loop with a hard upper bound.
94
- // We bail on epoch mismatch so closing the modal cancels the loop.
95
  while (Date.now() - start < POLL_TIMEOUT_MS) {
96
  if (epochRef.current !== myEpoch) return
97
  try {
@@ -124,8 +304,6 @@ export default function CreatorExportWizard({
124
  }
125
  await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
126
  }
127
- // Timed out: surface to the user, leave the job alone so they can
128
- // re-open the wizard later if it does eventually finish.
129
  if (epochRef.current === myEpoch) {
130
  setJob((prev) =>
131
  prev ? { ...prev, status: 'error', error: 'Render timed out' } : prev,
@@ -135,50 +313,57 @@ export default function CreatorExportWizard({
135
  [backendUrl, videoId],
136
  )
137
 
138
- const handleStart = useCallback(async () => {
139
- setSubmitError(null)
140
- epochRef.current += 1
141
- const myEpoch = epochRef.current
142
- setStep(2)
143
- setJob({ id: '', status: 'queued', progress: 0, kind })
144
- try {
145
- const r = await fetch(
146
- `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/mp4`,
147
- {
148
- method: 'POST',
149
- headers: { 'Content-Type': 'application/json', ...authHeaders() },
150
- body: JSON.stringify({ kind }),
151
- },
152
- )
153
- if (epochRef.current !== myEpoch) return
154
- if (!r.ok) {
155
- const text = await r.text().catch(() => '')
156
- const detail = text || `HTTP ${r.status}`
157
- setSubmitError(detail)
158
- setJob({ id: '', status: 'error', progress: 0, error: detail, kind })
159
- return
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
160
  }
161
- const j = await r.json()
162
- if (epochRef.current !== myEpoch) return
163
- setJob({
164
- id: j.id,
165
- status: j.status,
166
- progress: typeof j.progress === 'number' ? j.progress : 0,
167
- kind: j.kind,
168
- })
169
- void pollJob(j.id, myEpoch)
170
- } catch (err: any) {
171
- if (epochRef.current !== myEpoch) return
172
- const msg = String(err?.message || err)
173
- setSubmitError(msg)
174
- setJob({ id: '', status: 'error', progress: 0, error: msg, kind })
175
- }
176
- }, [backendUrl, videoId, kind, pollJob])
177
 
178
  if (!open) return null
179
 
180
  const submitDisabled = Boolean(disabledReason)
181
 
 
 
182
  return (
183
  <div
184
  className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
@@ -188,16 +373,26 @@ export default function CreatorExportWizard({
188
  aria-label="Export video"
189
  >
190
  <div
191
- className="w-full max-w-lg rounded-2xl border border-white/10 bg-[#0b0b0c] text-white shadow-2xl"
192
  onClick={(e) => e.stopPropagation()}
193
  >
194
- <div className="flex items-center justify-between border-b border-white/10 px-5 py-3">
 
195
  <div>
196
  <div className="text-sm font-semibold">Export video</div>
197
  {videoTitle ? (
198
  <div className="text-xs text-white/50 truncate max-w-[28rem]">{videoTitle}</div>
199
  ) : null}
200
  </div>
 
 
 
 
 
 
 
 
 
201
  <button
202
  type="button"
203
  onClick={onClose}
@@ -208,7 +403,8 @@ export default function CreatorExportWizard({
208
  </button>
209
  </div>
210
 
211
- <div className="px-5 py-5 space-y-4">
 
212
  {step === 1 && (
213
  <>
214
  <div className="text-xs text-white/60">
@@ -282,7 +478,7 @@ export default function CreatorExportWizard({
282
  </button>
283
  <button
284
  type="button"
285
- onClick={handleStart}
286
  disabled={submitDisabled}
287
  className={[
288
  'px-3 py-1.5 rounded-lg text-sm font-medium',
@@ -291,13 +487,172 @@ export default function CreatorExportWizard({
291
  : 'bg-white text-black hover:bg-white/90',
292
  ].join(' ')}
293
  >
294
- Render
295
  </button>
296
  </div>
297
  </>
298
  )}
299
 
300
  {step === 2 && (
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
301
  <>
302
  {job?.status === 'done' ? (
303
  <div className="space-y-4">
@@ -315,6 +670,7 @@ export default function CreatorExportWizard({
315
  </a>
316
  <div className="text-[11px] text-white/40 text-center">
317
  {job.kind === 'mp4_youtube' ? 'YouTube-optimized' : 'Plain MP4'}
 
318
  </div>
319
  </div>
320
  ) : job?.status === 'error' ? (
@@ -330,10 +686,10 @@ export default function CreatorExportWizard({
330
  </div>
331
  <button
332
  type="button"
333
- onClick={() => setStep(1)}
334
  className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
335
  >
336
- Try again
337
  </button>
338
  </div>
339
  ) : (
@@ -347,7 +703,9 @@ export default function CreatorExportWizard({
347
  <div className="h-2 rounded-full bg-white/10 overflow-hidden">
348
  <div
349
  className="h-full bg-cyan-400 transition-all"
350
- style={{ width: `${Math.max(2, Math.min(100, job?.progress || 0))}%` }}
 
 
351
  />
352
  </div>
353
  <div className="text-[11px] text-white/40 text-center">
 
1
  /**
2
+ * CreatorExportWizard β€” 4-step modal for exporting a Creator Studio video
3
+ * with in-browser Piper TTS narration.
4
  *
5
  * Flow:
6
+ * 1. Target β€” MP4 (edit/share) vs MP4 for YouTube.
7
+ * 2. Voiceover β€” Piper voice + rate + pitch + subtitle burn-in toggle,
8
+ * with a Preview button.
9
+ * 3. Generate β€” synthesize narration for every scene that has text
10
+ * (or re-synthesize all if the user checks "regenerate")
11
+ * and upload each WAV to the backend.
12
+ * 4. Render β€” POST /studio/videos/{id}/export/mp4, poll, download.
13
  *
14
  * Race protection:
15
+ * Every async op captures a per-mount epoch; closing the modal or
16
+ * starting a new render bumps the epoch and stale responses are
17
+ * dropped. Same pattern as App.tsx's account-switch guard.
 
18
  */
19
  import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
20
+ import {
21
+ AlertTriangle,
22
+ CheckCircle2,
23
+ Download,
24
+ Film,
25
+ Loader2,
26
+ Mic,
27
+ Play,
28
+ Square,
29
+ X,
30
+ Youtube,
31
+ } from 'lucide-react'
32
+
33
+ import {
34
+ listVoices,
35
+ synthesizeToBlob,
36
+ speak as piperSpeak,
37
+ stop as piperStop,
38
+ isSupported as piperSupported,
39
+ getSelectedVoiceId,
40
+ setSelectedVoiceId,
41
+ } from '../tts/piperTts'
42
+ import { DEFAULT_PIPER_VOICE_ID } from '../tts/piperVoices'
43
 
44
  export type ExportKind = 'mp4_plain' | 'mp4_youtube'
45
 
46
+ /** Minimum scene info the wizard needs. Matches the flat shape used
47
+ * inside CreatorStudioEditor. */
48
+ export interface WizardScene {
49
+ id: string
50
+ idx: number
51
+ narration: string
52
+ audioUrl: string | null
53
+ }
54
+
55
  interface Props {
56
  open: boolean
57
  onClose: () => void
58
  backendUrl: string
59
  videoId: string
60
+ /** Ordered scenes the render will concatenate. */
61
+ scenes: WizardScene[]
62
+ /** Display name shown in the modal header. */
63
  videoTitle?: string
64
  /** Disable submit when there are no scenes or scenes are still generating. */
65
  disabledReason?: string | null
 
77
 
78
  const POLL_INTERVAL_MS = 1000
79
  const POLL_TIMEOUT_MS = 10 * 60 * 1000 // 10 minutes
80
+ const PREVIEW_TEXT = 'Hello, this is a preview of your selected voice.'
81
 
82
  function authHeaders(): Record<string, string> {
83
  const token = localStorage.getItem('homepilot_auth_token') || ''
 
88
  return u.replace(/\/+$/, '')
89
  }
90
 
91
+ type Step = 1 | 2 | 3 | 4
92
+
93
  export default function CreatorExportWizard({
94
  open,
95
  onClose,
96
  backendUrl,
97
  videoId,
98
+ scenes,
99
  videoTitle,
100
  disabledReason,
101
  }: Props) {
102
+ // ── Step ─────────────────────────────────────────────────────────────────
103
+ const [step, setStep] = useState<Step>(1)
104
+
105
+ // ── Step 1 state ─────────────────────────────────────────────────────────
106
  const [kind, setKind] = useState<ExportKind>('mp4_youtube')
107
+
108
+ // ── Step 2 state ─────────────────────────────────────────────────────────
109
+ const voices = useMemo(() => listVoices(), [])
110
+ const [voiceId, setVoiceId] = useState<string>(() =>
111
+ piperSupported() ? getSelectedVoiceId() : DEFAULT_PIPER_VOICE_ID,
112
+ )
113
+ const [rate, setRate] = useState<number>(0.9)
114
+ const [pitch, setPitch] = useState<number>(1.0)
115
+ const [subtitlesBurnIn, setSubtitlesBurnIn] = useState<boolean>(true)
116
+ // Default: synthesize every scene that has narration text, even if an
117
+ // audioUrl already exists. The user's invariant is "audio from all
118
+ // scenes" at export time, so freshness beats speed by default.
119
+ // "Keep existing audio" below is the opt-in for re-exports of an
120
+ // unchanged project, where regenerating every take is wasted work.
121
+ const [keepExisting, setKeepExisting] = useState<boolean>(false)
122
+ const [previewing, setPreviewing] = useState<boolean>(false)
123
+ const [previewError, setPreviewError] = useState<string | null>(null)
124
+
125
+ // ── Step 3 state ─────────────────────────────────────────────────────────
126
+ const [genIdx, setGenIdx] = useState<number>(0)
127
+ const [genTotal, setGenTotal] = useState<number>(0)
128
+ const [genError, setGenError] = useState<string | null>(null)
129
+
130
+ // ── Step 4 state ─────────────────────────────────────────────────────────
131
  const [job, setJob] = useState<JobView | null>(null)
132
  const [submitError, setSubmitError] = useState<string | null>(null)
133
 
134
+ // Race protection.
 
135
  const epochRef = useRef(0)
136
 
137
+ // Reset when modal opens or closes.
138
  useEffect(() => {
139
  if (open) {
140
  setStep(1)
141
  setJob(null)
142
  setSubmitError(null)
143
+ setGenIdx(0)
144
+ setGenTotal(0)
145
+ setGenError(null)
146
+ setPreviewing(false)
147
+ setPreviewError(null)
148
  } else {
 
149
  epochRef.current += 1
150
+ try { piperStop() } catch { /* ignore */ }
151
  }
152
  }, [open])
153
 
154
+ // Persist voice selection so subsequent renders remember the pick.
155
+ useEffect(() => {
156
+ if (voiceId) setSelectedVoiceId(voiceId)
157
+ }, [voiceId])
158
+
159
  const downloadUrl = useMemo(() => {
160
  if (!job || job.status !== 'done') return ''
161
  const tok = localStorage.getItem('homepilot_auth_token') || ''
 
163
  return `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${job.id}/download${qp}`
164
  }, [job, backendUrl, videoId])
165
 
166
+ // ── Step 2 β€” Preview ─────────────────────────────────────────────────────
167
+
168
+ const handlePreview = useCallback(async () => {
169
+ setPreviewError(null)
170
+ if (previewing) {
171
+ try { piperStop() } catch { /* ignore */ }
172
+ setPreviewing(false)
173
+ return
174
+ }
175
+ if (!piperSupported()) {
176
+ setPreviewError('Piper requires a secure context (https) with Web Audio + OPFS support.')
177
+ return
178
+ }
179
+ setPreviewing(true)
180
+ try {
181
+ await piperSpeak(PREVIEW_TEXT, {
182
+ voiceId,
183
+ rate,
184
+ onEnd: () => setPreviewing(false),
185
+ onError: (err) => {
186
+ setPreviewError(String(err?.message || err))
187
+ setPreviewing(false)
188
+ },
189
+ })
190
+ } catch (err: any) {
191
+ setPreviewError(String(err?.message || err))
192
+ setPreviewing(false)
193
+ }
194
+ }, [previewing, voiceId, rate])
195
+
196
+ // ── Step 3 β€” Generate all scene narrations ───────────────────────────────
197
+
198
+ /** Scenes that need synthesis under the current toggle state.
199
+ * Default ("keep existing" off): every scene with narration text.
200
+ * Opt-in ("keep existing" on): only scenes that do NOT already
201
+ * have an uploaded audioUrl. */
202
+ const scenesToSynthesize = useMemo(
203
+ () =>
204
+ scenes.filter((s) => {
205
+ const hasText = (s.narration || '').trim().length > 0
206
+ if (!hasText) return false
207
+ if (keepExisting && s.audioUrl) return false
208
+ return true
209
+ }),
210
+ [scenes, keepExisting],
211
+ )
212
+
213
+ const handleGenerate = useCallback(async () => {
214
+ if (!piperSupported()) {
215
+ setGenError('Piper is not supported in this browser. Use a Chromium-based browser on https.')
216
+ return
217
+ }
218
+ epochRef.current += 1
219
+ const myEpoch = epochRef.current
220
+ setStep(3)
221
+ setGenError(null)
222
+ setGenIdx(0)
223
+ setGenTotal(scenesToSynthesize.length)
224
+
225
+ if (scenesToSynthesize.length === 0) {
226
+ // Nothing to synthesize β€” jump straight to render.
227
+ setStep(4)
228
+ void startRender(myEpoch)
229
+ return
230
+ }
231
+
232
+ for (let i = 0; i < scenesToSynthesize.length; i++) {
233
+ if (epochRef.current !== myEpoch) return
234
+ const scene = scenesToSynthesize[i]
235
+ try {
236
+ const blob = await synthesizeToBlob(scene.narration, { voiceId })
237
+ if (epochRef.current !== myEpoch) return
238
+
239
+ // Upload to the backend so render_mp4 can mix it in.
240
+ const fd = new FormData()
241
+ fd.append('file', blob, `${scene.id}.wav`)
242
+ const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/scenes/${scene.id}/narration`
243
+ const r = await fetch(url, {
244
+ method: 'POST',
245
+ headers: { ...authHeaders() },
246
+ body: fd,
247
+ })
248
+ if (epochRef.current !== myEpoch) return
249
+ if (!r.ok) {
250
+ const text = await r.text().catch(() => '')
251
+ throw new Error(text || `Upload failed (HTTP ${r.status})`)
252
+ }
253
+ } catch (err: any) {
254
+ if (epochRef.current !== myEpoch) return
255
+ setGenError(
256
+ `Scene ${scene.idx + 1}: ${String(err?.message || err)}`,
257
+ )
258
+ return
259
+ }
260
+ setGenIdx(i + 1)
261
+ }
262
+
263
+ if (epochRef.current !== myEpoch) return
264
+ // All done β€” kick off the render.
265
+ setStep(4)
266
+ void startRender(myEpoch)
267
+ }, [scenesToSynthesize, voiceId, backendUrl, videoId])
268
+
269
+ // ── Step 4 β€” Render + poll ───────────────────────────────────────────────
270
+
271
  const pollJob = useCallback(
272
  async (jobId: string, myEpoch: number) => {
273
  const url = `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/jobs/${jobId}`
274
  const start = Date.now()
 
 
275
  while (Date.now() - start < POLL_TIMEOUT_MS) {
276
  if (epochRef.current !== myEpoch) return
277
  try {
 
304
  }
305
  await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS))
306
  }
 
 
307
  if (epochRef.current === myEpoch) {
308
  setJob((prev) =>
309
  prev ? { ...prev, status: 'error', error: 'Render timed out' } : prev,
 
313
  [backendUrl, videoId],
314
  )
315
 
316
+ const startRender = useCallback(
317
+ async (myEpoch: number) => {
318
+ setSubmitError(null)
319
+ setJob({ id: '', status: 'queued', progress: 0, kind })
320
+ try {
321
+ const r = await fetch(
322
+ `${trimBackend(backendUrl)}/studio/videos/${videoId}/export/mp4`,
323
+ {
324
+ method: 'POST',
325
+ headers: { 'Content-Type': 'application/json', ...authHeaders() },
326
+ body: JSON.stringify({
327
+ kind,
328
+ audio_rate: rate,
329
+ audio_pitch: pitch,
330
+ subtitles: subtitlesBurnIn ? 'burn_in' : 'none',
331
+ }),
332
+ },
333
+ )
334
+ if (epochRef.current !== myEpoch) return
335
+ if (!r.ok) {
336
+ const text = await r.text().catch(() => '')
337
+ const detail = text || `HTTP ${r.status}`
338
+ setSubmitError(detail)
339
+ setJob({ id: '', status: 'error', progress: 0, error: detail, kind })
340
+ return
341
+ }
342
+ const j = await r.json()
343
+ if (epochRef.current !== myEpoch) return
344
+ setJob({
345
+ id: j.id,
346
+ status: j.status,
347
+ progress: typeof j.progress === 'number' ? j.progress : 0,
348
+ kind: j.kind,
349
+ })
350
+ void pollJob(j.id, myEpoch)
351
+ } catch (err: any) {
352
+ if (epochRef.current !== myEpoch) return
353
+ const msg = String(err?.message || err)
354
+ setSubmitError(msg)
355
+ setJob({ id: '', status: 'error', progress: 0, error: msg, kind })
356
  }
357
+ },
358
+ [backendUrl, videoId, kind, rate, pitch, subtitlesBurnIn, pollJob],
359
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
360
 
361
  if (!open) return null
362
 
363
  const submitDisabled = Boolean(disabledReason)
364
 
365
+ // ── UI ───────────────────────────────────────────────────────────────────
366
+
367
  return (
368
  <div
369
  className="fixed inset-0 z-50 flex items-center justify-center bg-black/70 backdrop-blur-sm p-4"
 
373
  aria-label="Export video"
374
  >
375
  <div
376
+ className="w-full max-w-xl rounded-2xl border border-white/10 bg-[#0b0b0c] text-white shadow-2xl max-h-[90vh] overflow-hidden flex flex-col"
377
  onClick={(e) => e.stopPropagation()}
378
  >
379
+ {/* Header */}
380
+ <div className="flex items-center justify-between border-b border-white/10 px-5 py-3 flex-shrink-0">
381
  <div>
382
  <div className="text-sm font-semibold">Export video</div>
383
  {videoTitle ? (
384
  <div className="text-xs text-white/50 truncate max-w-[28rem]">{videoTitle}</div>
385
  ) : null}
386
  </div>
387
+ <div className="flex items-center gap-3 text-[11px] text-white/40">
388
+ <span className={step >= 1 ? 'text-white/80' : ''}>1 Β· Target</span>
389
+ <span>β€Ί</span>
390
+ <span className={step >= 2 ? 'text-white/80' : ''}>2 Β· Voiceover</span>
391
+ <span>β€Ί</span>
392
+ <span className={step >= 3 ? 'text-white/80' : ''}>3 Β· Narration</span>
393
+ <span>β€Ί</span>
394
+ <span className={step >= 4 ? 'text-white/80' : ''}>4 Β· Render</span>
395
+ </div>
396
  <button
397
  type="button"
398
  onClick={onClose}
 
403
  </button>
404
  </div>
405
 
406
+ {/* Body */}
407
+ <div className="px-5 py-5 space-y-4 overflow-y-auto">
408
  {step === 1 && (
409
  <>
410
  <div className="text-xs text-white/60">
 
478
  </button>
479
  <button
480
  type="button"
481
+ onClick={() => setStep(2)}
482
  disabled={submitDisabled}
483
  className={[
484
  'px-3 py-1.5 rounded-lg text-sm font-medium',
 
487
  : 'bg-white text-black hover:bg-white/90',
488
  ].join(' ')}
489
  >
490
+ Next: Voiceover
491
  </button>
492
  </div>
493
  </>
494
  )}
495
 
496
  {step === 2 && (
497
+ <>
498
+ <div className="flex items-center gap-2 text-xs text-white/60">
499
+ <Mic size={14} className="text-cyan-300" />
500
+ Piper runs fully in-browser via WebAssembly. First use
501
+ downloads a voice model (~20 MB, cached).
502
+ </div>
503
+
504
+ <div className="space-y-3">
505
+ <label className="block text-[11px] uppercase tracking-wide text-white/50">
506
+ Piper voice
507
+ </label>
508
+ <select
509
+ value={voiceId}
510
+ onChange={(e) => setVoiceId(e.target.value)}
511
+ className="w-full px-3 py-2 rounded-lg bg-white/5 border border-white/10 text-sm text-white/90 focus:outline-none focus:border-white/30"
512
+ >
513
+ {voices.map((v) => (
514
+ <option key={v.id} value={v.id}>
515
+ {v.name} β€” {v.lang} Β· {v.gender} Β· {v.quality}
516
+ </option>
517
+ ))}
518
+ </select>
519
+ </div>
520
+
521
+ <div className="grid grid-cols-2 gap-4">
522
+ <div>
523
+ <div className="flex items-center justify-between text-[11px] uppercase tracking-wide text-white/50">
524
+ <span>Rate</span>
525
+ <span className="font-mono text-white/70">{rate.toFixed(2)}</span>
526
+ </div>
527
+ <input
528
+ type="range"
529
+ min={0.5}
530
+ max={2.0}
531
+ step={0.05}
532
+ value={rate}
533
+ onChange={(e) => setRate(Number(e.target.value))}
534
+ className="w-full accent-cyan-400"
535
+ />
536
+ </div>
537
+ <div>
538
+ <div className="flex items-center justify-between text-[11px] uppercase tracking-wide text-white/50">
539
+ <span>Pitch</span>
540
+ <span className="font-mono text-white/70">{pitch.toFixed(2)}</span>
541
+ </div>
542
+ <input
543
+ type="range"
544
+ min={0.5}
545
+ max={2.0}
546
+ step={0.05}
547
+ value={pitch}
548
+ onChange={(e) => setPitch(Number(e.target.value))}
549
+ className="w-full accent-cyan-400"
550
+ />
551
+ </div>
552
+ </div>
553
+
554
+ <div className="flex items-center gap-2">
555
+ <button
556
+ type="button"
557
+ onClick={handlePreview}
558
+ className="inline-flex items-center gap-2 px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15 border border-white/10"
559
+ >
560
+ {previewing ? <Square size={14} /> : <Play size={14} />}
561
+ {previewing ? 'Stop preview' : 'Preview voice'}
562
+ </button>
563
+ <span className="text-[11px] text-white/40 truncate">{PREVIEW_TEXT}</span>
564
+ </div>
565
+
566
+ {previewError ? (
567
+ <div className="text-[11px] text-red-300/80">{previewError}</div>
568
+ ) : null}
569
+
570
+ <label className="flex items-center gap-2 text-sm text-white/80 cursor-pointer">
571
+ <input
572
+ type="checkbox"
573
+ checked={subtitlesBurnIn}
574
+ onChange={(e) => setSubtitlesBurnIn(e.target.checked)}
575
+ className="accent-cyan-400"
576
+ />
577
+ Burn subtitles into the video (from scene narration text)
578
+ </label>
579
+
580
+ <label className="flex items-center gap-2 text-sm text-white/80 cursor-pointer">
581
+ <input
582
+ type="checkbox"
583
+ checked={keepExisting}
584
+ onChange={(e) => setKeepExisting(e.target.checked)}
585
+ className="accent-cyan-400"
586
+ />
587
+ Keep existing narration audio where present
588
+ <span className="text-[11px] text-white/40">(skip scenes that already have a take)</span>
589
+ </label>
590
+
591
+ <div className="text-[11px] text-white/40">
592
+ {scenesToSynthesize.length === 0
593
+ ? 'No scenes have narration text β€” the export will have silent audio.'
594
+ : `${scenesToSynthesize.length} of ${scenes.length} scene${scenes.length === 1 ? '' : 's'} will be synthesized with the selected voice.`}
595
+ </div>
596
+
597
+ <div className="flex justify-between gap-2 pt-2">
598
+ <button
599
+ type="button"
600
+ onClick={() => setStep(1)}
601
+ className="px-3 py-1.5 rounded-lg text-sm text-white/70 hover:bg-white/10"
602
+ >
603
+ Back
604
+ </button>
605
+ <button
606
+ type="button"
607
+ onClick={handleGenerate}
608
+ className="px-3 py-1.5 rounded-lg text-sm font-medium bg-white text-black hover:bg-white/90"
609
+ >
610
+ {scenesToSynthesize.length === 0 ? 'Skip & render' : 'Generate narration'}
611
+ </button>
612
+ </div>
613
+ </>
614
+ )}
615
+
616
+ {step === 3 && (
617
+ <>
618
+ <div className="flex items-center gap-2 text-white/80">
619
+ <Loader2 size={16} className="animate-spin" />
620
+ <span className="text-sm">Synthesizing narration…</span>
621
+ </div>
622
+ <div className="h-2 rounded-full bg-white/10 overflow-hidden">
623
+ <div
624
+ className="h-full bg-cyan-400 transition-all"
625
+ style={{
626
+ width: `${genTotal === 0 ? 0 : Math.round((genIdx / genTotal) * 100)}%`,
627
+ }}
628
+ />
629
+ </div>
630
+ <div className="text-[11px] text-white/40 text-center">
631
+ {genIdx} of {genTotal} scenes
632
+ </div>
633
+
634
+ {genError ? (
635
+ <div className="space-y-3">
636
+ <div className="flex items-start gap-2 rounded-lg border border-red-400/30 bg-red-400/10 p-3 text-[12px] text-red-200">
637
+ <AlertTriangle size={14} className="mt-0.5 flex-shrink-0" />
638
+ <div>
639
+ <div className="font-medium mb-0.5">Narration synthesis failed</div>
640
+ <div className="text-red-300/80 break-words">{genError}</div>
641
+ </div>
642
+ </div>
643
+ <button
644
+ type="button"
645
+ onClick={() => setStep(2)}
646
+ className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
647
+ >
648
+ Back to voiceover
649
+ </button>
650
+ </div>
651
+ ) : null}
652
+ </>
653
+ )}
654
+
655
+ {step === 4 && (
656
  <>
657
  {job?.status === 'done' ? (
658
  <div className="space-y-4">
 
670
  </a>
671
  <div className="text-[11px] text-white/40 text-center">
672
  {job.kind === 'mp4_youtube' ? 'YouTube-optimized' : 'Plain MP4'}
673
+ {subtitlesBurnIn ? ' Β· subtitles burned in' : ''}
674
  </div>
675
  </div>
676
  ) : job?.status === 'error' ? (
 
686
  </div>
687
  <button
688
  type="button"
689
+ onClick={() => setStep(2)}
690
  className="w-full px-3 py-1.5 rounded-lg text-sm bg-white/10 hover:bg-white/15"
691
  >
692
+ Back to voiceover
693
  </button>
694
  </div>
695
  ) : (
 
703
  <div className="h-2 rounded-full bg-white/10 overflow-hidden">
704
  <div
705
  className="h-full bg-cyan-400 transition-all"
706
+ style={{
707
+ width: `${Math.max(2, Math.min(100, job?.progress || 0))}%`,
708
+ }}
709
  />
710
  </div>
711
  <div className="text-[11px] text-white/40 text-center">
frontend/src/ui/components/TtsEngineSection.tsx ADDED
@@ -0,0 +1,279 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Schema-driven Settings section for the active TTS engine.
3
+ *
4
+ * This component is deliberately self-contained so SettingsPanel can mount
5
+ * it with one line and stay agnostic of which engines are installed.
6
+ *
7
+ * Behavior:
8
+ * - Dropdown to pick the active engine (only ``isAvailable()`` ones).
9
+ * - Below: controls rendered from the active provider's
10
+ * ``getSettingsSchema()`` β€” ``select`` / ``range`` / ``toggle``.
11
+ * - The Web Speech provider's ``voiceId`` schema has a placeholder
12
+ * options list; we merge in the browser's live voices at render
13
+ * time so the user sees their actual installed voices.
14
+ * - Every edit persists to the registry's per-user scoped settings
15
+ * bucket AND to ``value`` via ``onChangeDraft`` so the existing
16
+ * Voice Assistant code continues to read ``value.selectedVoice``.
17
+ *
18
+ * Purely additive: the existing System Voice select stays untouched
19
+ * above this component, and this section only appears when the user
20
+ * opts into it by picking a non-default engine (or leaves it on
21
+ * ``web-speech-api`` where we render the same controls the old UI did).
22
+ */
23
+
24
+ import React, { useEffect, useMemo, useState } from 'react'
25
+ import {
26
+ getActiveTtsEngineId,
27
+ listTtsProviders,
28
+ onActiveTtsEngineChange,
29
+ readTtsProviderSettings,
30
+ setActiveTtsEngine,
31
+ writeTtsProviderSettings,
32
+ } from '../tts'
33
+ import type { SettingsField, TtsProvider } from '../tts'
34
+
35
+ interface Props {
36
+ /** Optional: used by the Web Speech engine to populate its voice
37
+ * dropdown with the browser's live voices. Passing undefined is
38
+ * safe β€” we fall back to ``getVoices()``. */
39
+ systemVoices?: readonly SpeechSynthesisVoice[]
40
+ }
41
+
42
+ function _fieldValue(schema: SettingsField, saved: Record<string, unknown>): string | number | boolean {
43
+ const v = saved[schema.key]
44
+ if (schema.kind === 'range') {
45
+ return typeof v === 'number' ? v : schema.defaultValue
46
+ }
47
+ if (schema.kind === 'toggle') {
48
+ return typeof v === 'boolean' ? v : schema.defaultValue
49
+ }
50
+ return typeof v === 'string' ? v : schema.defaultValue
51
+ }
52
+
53
+ function _mergeWebSpeechOptions(
54
+ field: SettingsField,
55
+ voices: readonly SpeechSynthesisVoice[],
56
+ ): SettingsField {
57
+ if (field.kind !== 'select' || field.key !== 'voiceId' || voices.length === 0) {
58
+ return field
59
+ }
60
+ const live = voices.map((v) => ({
61
+ value: v.voiceURI || v.name,
62
+ label: `${v.name} (${v.lang})`,
63
+ }))
64
+ return { ...field, options: [...field.options, ...live] }
65
+ }
66
+
67
+ export default function TtsEngineSection({ systemVoices }: Props): JSX.Element {
68
+ const [activeId, setActiveIdState] = useState<string>(() => getActiveTtsEngineId())
69
+ // Re-render when another component swaps the active engine.
70
+ useEffect(() => onActiveTtsEngineChange((id) => setActiveIdState(id)), [])
71
+
72
+ const providers = useMemo<readonly TtsProvider[]>(
73
+ () => listTtsProviders().filter((p) => p.isAvailable()),
74
+ [activeId],
75
+ )
76
+
77
+ const active = useMemo<TtsProvider | undefined>(
78
+ () => providers.find((p) => p.id === activeId),
79
+ [providers, activeId],
80
+ )
81
+
82
+ // Saved settings blob for the active provider.
83
+ const [settings, setSettings] = useState<Record<string, unknown>>(
84
+ () => readTtsProviderSettings(activeId),
85
+ )
86
+ useEffect(() => {
87
+ setSettings(readTtsProviderSettings(activeId))
88
+ }, [activeId])
89
+
90
+ // When the default engine is selected the host panel already renders
91
+ // a dedicated "Assistant Voice" / "System Voice" dropdown above us
92
+ // that covers voice + rate + pitch for Web Speech. Rendering our
93
+ // schema-driven twin on top of it is pure duplication β€” skip it
94
+ // unless the user has opted into a non-default engine (Piper etc.).
95
+ const isDefaultEngine = activeId === 'web-speech-api'
96
+ const schema = active && !isDefaultEngine ? active.getSettingsSchema() : []
97
+
98
+ // Test-voice state: the button speaks through whichever engine is
99
+ // active right now so the user can hear their pick without leaving
100
+ // Settings. Mirrors the Preview button in the Creator Studio wizard.
101
+ const [testing, setTesting] = useState(false)
102
+ const [testError, setTestError] = useState<string | null>(null)
103
+ const onTest = () => {
104
+ setTestError(null)
105
+ if (!active) return
106
+ if (testing) {
107
+ try { active.stop() } catch { /* ignore */ }
108
+ setTesting(false)
109
+ return
110
+ }
111
+ setTesting(true)
112
+ const voiceId = typeof settings.voiceId === 'string' ? settings.voiceId : undefined
113
+ const rate = typeof settings.rate === 'number' ? settings.rate : undefined
114
+ const pitch = typeof settings.pitch === 'number' ? settings.pitch : undefined
115
+ active
116
+ .speak('Hello, this is a preview of your selected voice.', {
117
+ voiceId,
118
+ rate,
119
+ pitch,
120
+ onEnd: () => setTesting(false),
121
+ onError: (err) => {
122
+ setTestError(String(err?.message || err))
123
+ setTesting(false)
124
+ },
125
+ })
126
+ .catch((err) => {
127
+ setTestError(String(err?.message || err))
128
+ setTesting(false)
129
+ })
130
+ }
131
+
132
+ const onChangeField = (key: string, value: string | number | boolean) => {
133
+ const next = { ...settings, [key]: value }
134
+ setSettings(next)
135
+ writeTtsProviderSettings(activeId, { [key]: value })
136
+ }
137
+
138
+ return (
139
+ <div className="border-t border-white/5 pt-3">
140
+ <div className="text-[11px] uppercase tracking-wider text-white/40 mb-2 font-semibold">
141
+ TTS Engine
142
+ </div>
143
+
144
+ <div className="space-y-3">
145
+ <div>
146
+ <label className="block text-[10px] text-white/50 mb-2">Engine</label>
147
+ <select
148
+ className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
149
+ value={activeId}
150
+ onChange={(e) => setActiveTtsEngine(e.target.value)}
151
+ >
152
+ {providers.map((p) => (
153
+ <option key={p.id} value={p.id}>
154
+ {p.displayName}
155
+ </option>
156
+ ))}
157
+ </select>
158
+ {active?.id === 'piper-wasm' && (
159
+ <div className="mt-1 text-[10px] text-white/40 leading-relaxed">
160
+ Piper runs fully in-browser via WebAssembly. First use
161
+ downloads a voice model (~20 MB, cached). Falls back to the
162
+ HomePilot mirror when the upstream CDN is unreachable.
163
+ </div>
164
+ )}
165
+ </div>
166
+
167
+ {active && !isDefaultEngine ? (
168
+ schema.map((field) => {
169
+ const resolved =
170
+ active.id === 'web-speech-api' && field.kind === 'select' && field.key === 'voiceId'
171
+ ? _mergeWebSpeechOptions(
172
+ field,
173
+ systemVoices ??
174
+ (typeof window !== 'undefined' && 'speechSynthesis' in window
175
+ ? window.speechSynthesis.getVoices()
176
+ : []),
177
+ )
178
+ : field
179
+ const value = _fieldValue(resolved, settings)
180
+
181
+ if (resolved.kind === 'select') {
182
+ return (
183
+ <div key={resolved.key}>
184
+ <label className="block text-[10px] text-white/50 mb-2">
185
+ {resolved.label}
186
+ </label>
187
+ <select
188
+ className="w-full bg-black border border-white/10 rounded-lg px-3 py-2 text-xs text-white"
189
+ value={String(value)}
190
+ onChange={(e) => onChangeField(resolved.key, e.target.value)}
191
+ >
192
+ {resolved.options.map((o) => (
193
+ <option key={o.value} value={o.value}>
194
+ {o.label}
195
+ </option>
196
+ ))}
197
+ </select>
198
+ {resolved.description ? (
199
+ <div className="mt-1 text-[10px] text-white/40">{resolved.description}</div>
200
+ ) : null}
201
+ </div>
202
+ )
203
+ }
204
+
205
+ if (resolved.kind === 'range') {
206
+ return (
207
+ <div key={resolved.key}>
208
+ <div className="flex items-center justify-between">
209
+ <label className="text-[10px] text-white/50">{resolved.label}</label>
210
+ <span className="text-[10px] text-white/60 font-mono">
211
+ {typeof value === 'number' ? value.toFixed(2) : value}
212
+ </span>
213
+ </div>
214
+ <input
215
+ type="range"
216
+ min={resolved.min}
217
+ max={resolved.max}
218
+ step={resolved.step}
219
+ value={typeof value === 'number' ? value : resolved.defaultValue}
220
+ onChange={(e) => onChangeField(resolved.key, Number(e.target.value))}
221
+ className="w-full accent-cyan-400"
222
+ />
223
+ </div>
224
+ )
225
+ }
226
+
227
+ // toggle
228
+ return (
229
+ <label key={resolved.key} className="flex items-center gap-2 cursor-pointer">
230
+ <input
231
+ type="checkbox"
232
+ checked={Boolean(value)}
233
+ onChange={(e) => onChangeField(resolved.key, e.target.checked)}
234
+ className="w-4 h-4 rounded"
235
+ />
236
+ <span className="text-xs text-white">{resolved.label}</span>
237
+ </label>
238
+ )
239
+ })
240
+ ) : !active ? (
241
+ <div className="text-[10px] text-white/40">
242
+ No TTS engine available in this environment.
243
+ </div>
244
+ ) : null}
245
+
246
+ {active && !isDefaultEngine && !active.capabilities.pitch && (
247
+ <div className="text-[10px] text-white/40 italic">
248
+ The {active.displayName.split(' (')[0]} engine does not support a pitch control in
249
+ playback. The Creator Studio export applies pitch post-synthesis via ffmpeg.
250
+ </div>
251
+ )}
252
+
253
+ {/* Test voice button β€” positioned AFTER the voice + rate + pitch
254
+ controls so users pick their voice first, then preview it
255
+ (best-practice: action sits at the end of the configuration
256
+ flow). Always visible; mirrors the "Preview voice" affordance
257
+ in the Creator Studio export wizard. */}
258
+ {active && (
259
+ <div className="flex items-center gap-2 pt-1">
260
+ <button
261
+ type="button"
262
+ onClick={onTest}
263
+ className="text-[11px] px-3 py-1.5 rounded-lg bg-white/10 hover:bg-white/15 border border-white/10 text-white/80"
264
+ aria-pressed={testing}
265
+ >
266
+ {testing ? 'Stop' : 'Test voice'}
267
+ </button>
268
+ <span className="text-[10px] text-white/40">
269
+ Hello, this is a preview of your selected voice.
270
+ </span>
271
+ </div>
272
+ )}
273
+ {testError ? (
274
+ <div className="text-[10px] text-red-300/80">{testError}</div>
275
+ ) : null}
276
+ </div>
277
+ </div>
278
+ )
279
+ }
frontend/src/ui/lib/backendUrl.test.ts ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { afterEach, describe, expect, it, vi } from 'vitest'
2
+ import { resolveBackendUrl } from './backendUrl'
3
+
4
+ afterEach(() => {
5
+ window.localStorage.clear()
6
+ vi.unstubAllGlobals()
7
+ })
8
+
9
+ describe('resolveBackendUrl', () => {
10
+ it('falls back to localhost:8000 when stored backend points to local vite dev port', () => {
11
+ vi.stubGlobal('location', {
12
+ hostname: 'localhost',
13
+ origin: 'http://localhost:3000',
14
+ port: '3000',
15
+ protocol: 'http:',
16
+ })
17
+
18
+ window.localStorage.setItem('homepilot_backend_url', 'http://localhost:3000')
19
+
20
+ expect(resolveBackendUrl()).toBe('http://localhost:8000')
21
+ })
22
+
23
+ it('keeps stored localhost backend when it targets a non-vite port', () => {
24
+ vi.stubGlobal('location', {
25
+ hostname: 'localhost',
26
+ origin: 'http://localhost:3000',
27
+ port: '3000',
28
+ protocol: 'http:',
29
+ })
30
+
31
+ window.localStorage.setItem('homepilot_backend_url', 'http://localhost:8010')
32
+
33
+ expect(resolveBackendUrl()).toBe('http://localhost:8010')
34
+ })
35
+ })
frontend/src/ui/lib/backendUrl.ts CHANGED
@@ -19,7 +19,12 @@
19
  */
20
 
21
  const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1"]);
22
- const DEV_VITE_PORTS = new Set(["5173", "5174"]);
 
 
 
 
 
23
 
24
  function isLocalHost(hostname: string): boolean {
25
  return LOCAL_HOSTS.has(hostname);
@@ -70,13 +75,25 @@ export function resolveBackendUrl(override?: string | null): string {
70
  if (stored && stored.trim()) {
71
  const clean = stripTrailingSlash(stored.trim());
72
  const pageIsRemote = !isLocalHost(window.location.hostname);
73
- const storedHost = parsedHostname(clean);
 
 
74
  const storedIsLocal = storedHost !== null && isLocalHost(storedHost);
75
  if (pageIsRemote && storedIsLocal) {
76
  // Stale setting from a previous local dev session. Ignore it and
77
  // let getDefaultBackendUrl() take over (same origin).
78
  return getDefaultBackendUrl();
79
  }
 
 
 
 
 
 
 
 
 
 
80
  return clean;
81
  }
82
  } catch {
 
19
  */
20
 
21
  const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "0.0.0.0", "::1"]);
22
+ // Vite dev ports used across the repo. The Makefile's `make start` target
23
+ // launches vite with --port 3000; earlier tooling used Vite's defaults
24
+ // 5173/5174. All of them must route API traffic to the backend at :8000
25
+ // rather than to the Vite origin (which would SPA-fallback every request
26
+ // to index.html and break JSON.parse).
27
+ const DEV_VITE_PORTS = new Set(["3000", "5173", "5174"]);
28
 
29
  function isLocalHost(hostname: string): boolean {
30
  return LOCAL_HOSTS.has(hostname);
 
75
  if (stored && stored.trim()) {
76
  const clean = stripTrailingSlash(stored.trim());
77
  const pageIsRemote = !isLocalHost(window.location.hostname);
78
+ const storedUrl = (() => { try { return new URL(clean); } catch { return null; } })();
79
+ const storedHost = storedUrl?.hostname ?? null;
80
+ const storedPort = storedUrl?.port ?? "";
81
  const storedIsLocal = storedHost !== null && isLocalHost(storedHost);
82
  if (pageIsRemote && storedIsLocal) {
83
  // Stale setting from a previous local dev session. Ignore it and
84
  // let getDefaultBackendUrl() take over (same origin).
85
  return getDefaultBackendUrl();
86
  }
87
+ // Stale setting pointing at a Vite DEV port on localhost (3000 / 5173
88
+ // / 5174). The frontend origin serves the SPA β€” POSTing JSON there
89
+ // SPA-falls-back to index.html and the caller sees a 404, which the
90
+ // voice-call path reads as "feature unavailable" and flips to the
91
+ // chat-REST fallback. Redirect to :8000 (the default backend port)
92
+ // so the stored preference doesn't silently break voice-call
93
+ // session creation.
94
+ if (storedIsLocal && DEV_VITE_PORTS.has(storedPort)) {
95
+ return "http://localhost:8000";
96
+ }
97
  return clean;
98
  }
99
  } catch {
frontend/src/ui/phone/PostCallCard.test.tsx ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * Unit tests for PostCallCard.
3
+ *
4
+ * Anchored to the variant Γ— missed matrix in Β§ 3 of the handoff.
5
+ * Each test names the invariant it protects.
6
+ */
7
+ import React from 'react'
8
+ import { describe, expect, it, vi } from 'vitest'
9
+ import { render, screen, fireEvent } from '@testing-library/react'
10
+
11
+ import PostCallCard, { postCallCardInternals } from './PostCallCard'
12
+
13
+ const { formatDuration } = postCallCardInternals
14
+
15
+ describe('formatDuration', () => {
16
+ it('renders seconds under a minute', () => {
17
+ expect(formatDuration(5)).toBe('5s')
18
+ expect(formatDuration(59)).toBe('59s')
19
+ })
20
+ it('rounds to whole minutes when exact', () => {
21
+ expect(formatDuration(60)).toBe('1 min')
22
+ expect(formatDuration(720)).toBe('12 min')
23
+ })
24
+ it('renders minutes + residual seconds', () => {
25
+ expect(formatDuration(64)).toBe('1 min 4s')
26
+ })
27
+ })
28
+
29
+ describe('PostCallCard β€” expand variant (default)', () => {
30
+ it('renders duration + persona name', () => {
31
+ render(<PostCallCard durationSec={42} personaName="Vesper" />)
32
+ expect(screen.getByText(/Call with Vesper/i)).toBeInTheDocument()
33
+ expect(screen.getByText(/42s/)).toBeInTheDocument()
34
+ })
35
+
36
+ it('hides the transcript button when no transcript is supplied', () => {
37
+ render(<PostCallCard durationSec={42} personaName="Vesper" />)
38
+ expect(screen.queryByRole('button', { name: /view transcript/i })).not.toBeInTheDocument()
39
+ })
40
+
41
+ it('shows the transcript button when a transcript is supplied', () => {
42
+ render(
43
+ <PostCallCard
44
+ durationSec={42}
45
+ personaName="Vesper"
46
+ transcript={[{ who: 'user', text: 'hello' }]}
47
+ />,
48
+ )
49
+ expect(screen.getByRole('button', { name: /view transcript/i })).toBeInTheDocument()
50
+ })
51
+
52
+ it('expands the transcript inline (NOT a modal) when clicked', () => {
53
+ render(
54
+ <PostCallCard
55
+ durationSec={42}
56
+ personaName="Vesper"
57
+ transcript={[
58
+ { who: 'user', text: 'hello' },
59
+ { who: 'assistant', text: 'hey there' },
60
+ ]}
61
+ />,
62
+ )
63
+ fireEvent.click(screen.getByRole('button', { name: /view transcript/i }))
64
+ expect(screen.getByText('hey there')).toBeInTheDocument()
65
+ // The only role=dialog in the tree would be a modal; assert none.
66
+ expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
67
+ })
68
+
69
+ it('collapses the transcript on a second click', () => {
70
+ render(
71
+ <PostCallCard
72
+ durationSec={42}
73
+ personaName="Vesper"
74
+ transcript={[{ who: 'assistant', text: 'hey there' }]}
75
+ />,
76
+ )
77
+ const btn = screen.getByRole('button', { name: /view transcript/i })
78
+ fireEvent.click(btn)
79
+ expect(screen.getByText('hey there')).toBeInTheDocument()
80
+ fireEvent.click(screen.getByRole('button', { name: /collapse transcript/i }))
81
+ expect(screen.queryByText('hey there')).not.toBeInTheDocument()
82
+ })
83
+
84
+ it('fires onResume when Resume call is clicked', () => {
85
+ const onResume = vi.fn()
86
+ render(
87
+ <PostCallCard durationSec={42} personaName="Vesper" onResume={onResume} />,
88
+ )
89
+ fireEvent.click(screen.getByRole('button', { name: /resume call/i }))
90
+ expect(onResume).toHaveBeenCalledTimes(1)
91
+ })
92
+ })
93
+
94
+ describe('PostCallCard β€” highlights variant', () => {
95
+ it('shows the summary line and a "Full transcript" button when handler supplied', () => {
96
+ const onOpen = vi.fn()
97
+ render(
98
+ <PostCallCard
99
+ durationSec={600}
100
+ personaName="Vesper"
101
+ variant="highlights"
102
+ summary="You talked about Lisbon and made plans for Friday."
103
+ onOpenFullTranscript={onOpen}
104
+ />,
105
+ )
106
+ expect(screen.getByText(/Vesper remembers/i)).toBeInTheDocument()
107
+ expect(
108
+ screen.getByText(/You talked about Lisbon and made plans for Friday/i),
109
+ ).toBeInTheDocument()
110
+ fireEvent.click(screen.getByRole('button', { name: /open full transcript/i }))
111
+ expect(onOpen).toHaveBeenCalledTimes(1)
112
+ })
113
+
114
+ it('omits the "Full transcript" button when no handler is supplied', () => {
115
+ render(
116
+ <PostCallCard
117
+ durationSec={600}
118
+ personaName="Vesper"
119
+ variant="highlights"
120
+ summary="…"
121
+ />,
122
+ )
123
+ expect(
124
+ screen.queryByRole('button', { name: /open full transcript/i }),
125
+ ).not.toBeInTheDocument()
126
+ })
127
+ })
128
+
129
+ describe('PostCallCard β€” missed variant', () => {
130
+ it('swaps Resume β†’ Call back and omits secondary CTA', () => {
131
+ const onCallBack = vi.fn()
132
+ render(
133
+ <PostCallCard
134
+ durationSec={0}
135
+ personaName="Vesper"
136
+ missed
137
+ onCallBack={onCallBack}
138
+ />,
139
+ )
140
+ expect(screen.getByText(/Missed call/i)).toBeInTheDocument()
141
+ fireEvent.click(screen.getByRole('button', { name: /call back/i }))
142
+ expect(onCallBack).toHaveBeenCalledTimes(1)
143
+ // No secondary button at all in the missed variant.
144
+ expect(screen.queryByRole('button', { name: /view transcript/i })).not.toBeInTheDocument()
145
+ expect(screen.queryByRole('button', { name: /full transcript/i })).not.toBeInTheDocument()
146
+ })
147
+
148
+ it('falls back to onResume when onCallBack is omitted', () => {
149
+ const onResume = vi.fn()
150
+ render(
151
+ <PostCallCard durationSec={0} personaName="Vesper" missed onResume={onResume} />,
152
+ )
153
+ fireEvent.click(screen.getByRole('button', { name: /call back/i }))
154
+ expect(onResume).toHaveBeenCalledTimes(1)
155
+ })
156
+ })
frontend/src/ui/phone/PostCallCard.tsx ADDED
@@ -0,0 +1,384 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * PostCallCard β€” inline "we just talked" record rendered directly
3
+ * in the chat thread, NOT as a modal. Product direction from the
4
+ * engineering handoff:
5
+ *
6
+ * "You are building a conversational / companion experience,
7
+ * not a compliance tool. β†’ Use inline card + expandable
8
+ * transcript."
9
+ *
10
+ * Three variants covering the full (variant Γ— missed) matrix from
11
+ * the handoff's Β§ 3:
12
+ *
13
+ * expand β€” default. Duration + time + [Resume call] +
14
+ * [View transcript β–Ύ]. Tapping the latter expands
15
+ * the transcript INLINE inside the same card (no
16
+ * modal). Keeps the call record as an event in
17
+ * the conversation the user can glance at later.
18
+ * highlights β€” same shell, but the body is a short italic
19
+ * "Vesper remembers…" summary + a [Full transcript]
20
+ * link. For sessions where the call was long and
21
+ * the full transcript is too long to inline.
22
+ * missed β€” red tint, phone-slash icon, single [Call back]
23
+ * CTA. Set via ``missed`` prop (orthogonal to
24
+ * variant; a missed call is always rendered as the
25
+ * missed variant, others ignored).
26
+ *
27
+ * This component replaces the prior enterprise-neutral
28
+ * CallMemoryCard. The enterprise form (flat surface, \"Voice session
29
+ * completed\" copy) is preserved as a setting-free default by
30
+ * passing only durationSec + endedAt; richer props turn on the
31
+ * conversational surface.
32
+ */
33
+
34
+ import React, { useState } from 'react'
35
+ import {
36
+ IconPhone,
37
+ IconPhoneMissed,
38
+ IconSparkle,
39
+ } from './icons'
40
+ import { CALL, POST_CALL } from './tokens'
41
+
42
+ // ── Props ────────────────────────────────────────────────────────
43
+
44
+ export type PostCallVariant = 'expand' | 'highlights'
45
+
46
+ export interface TranscriptLine {
47
+ who: 'user' | 'assistant'
48
+ text: string
49
+ }
50
+
51
+ export interface PostCallCardProps {
52
+ /** Live-phase duration in seconds. Rendered as "12s" / "1m 4s" /
53
+ * "12 min" depending on magnitude. */
54
+ durationSec: number
55
+ /** Epoch-ms timestamp of when the call ended. Rendered as
56
+ * "8:14 PM" via toLocaleTimeString. */
57
+ endedAt?: number
58
+ /** The persona the user was talking to. Drives the body text
59
+ * ("Call with Vesper"). Default "Assistant". */
60
+ personaName?: string
61
+ /** Variant β€” see module docstring. */
62
+ variant?: PostCallVariant
63
+ /** Missed-call variant β€” tints red, shows phone-slash icon,
64
+ * swaps the CTAs. */
65
+ missed?: boolean
66
+ /** Optional transcript. When present AND variant='expand',
67
+ * enables the inline expand button. */
68
+ transcript?: TranscriptLine[]
69
+ /** Optional short summary for variant='highlights'
70
+ * (e.g. "You talked about Lisbon and made plans for Friday"). */
71
+ summary?: string
72
+ /** "Resume call" click β€” re-open the call overlay with
73
+ * skipDialing=true. */
74
+ onResume?: () => void
75
+ /** "Call back" click on the missed variant. Defaults to the same
76
+ * as onResume when omitted. */
77
+ onCallBack?: () => void
78
+ /** "Full transcript" click on the highlights variant β€” typically
79
+ * opens a separate route or sheet. Optional; button hides when
80
+ * handler isn't provided. */
81
+ onOpenFullTranscript?: () => void
82
+ }
83
+
84
+ // ── Helpers ──────────────────────────────────────────────────────
85
+
86
+ function formatDuration(s: number): string {
87
+ if (s < 60) return `${s}s`
88
+ const m = Math.floor(s / 60)
89
+ const r = s % 60
90
+ if (r === 0) return m < 60 ? `${m} min` : `${Math.floor(m / 60)}h ${m % 60}m`
91
+ return `${m} min ${r}s`
92
+ }
93
+
94
+ function formatClock(endedAt: number): string {
95
+ return new Date(endedAt).toLocaleTimeString(undefined, {
96
+ hour: 'numeric',
97
+ minute: '2-digit',
98
+ })
99
+ }
100
+
101
+ // ── Component ────────────────────────────────────────────────────
102
+
103
+ const PostCallCard: React.FC<PostCallCardProps> = ({
104
+ durationSec,
105
+ endedAt,
106
+ personaName = 'Assistant',
107
+ variant = 'expand',
108
+ missed = false,
109
+ transcript,
110
+ summary,
111
+ onResume,
112
+ onCallBack,
113
+ onOpenFullTranscript,
114
+ }) => {
115
+ const [expanded, setExpanded] = useState(false)
116
+ const clock = endedAt ? formatClock(endedAt) : ''
117
+ const dur = formatDuration(durationSec)
118
+ const accent = missed ? CALL.danger : CALL.rose
119
+ const hasTranscript = !!(transcript && transcript.length > 0)
120
+
121
+ // Top row β€” icon, title, sub.
122
+ const subtitle = missed
123
+ ? clock
124
+ ? `${clock} Β· tap to call back`
125
+ : 'tap to call back'
126
+ : clock
127
+ ? `${dur} Β· ${clock}`
128
+ : dur
129
+
130
+ const titleText = missed
131
+ ? `Missed call Β· ${personaName}`
132
+ : `Call with ${personaName}`
133
+
134
+ return (
135
+ <div
136
+ role="note"
137
+ aria-label={missed ? `Missed call from ${personaName}` : `Call with ${personaName}`}
138
+ className="hp-fade-in"
139
+ style={{
140
+ width: POST_CALL.cardWidth,
141
+ maxWidth: '100%',
142
+ borderRadius: POST_CALL.cardRadius,
143
+ overflow: 'hidden',
144
+ background: missed
145
+ ? `linear-gradient(180deg, color-mix(in oklch, ${accent} 12%, transparent) 0%, rgba(245,236,255,0.03) 100%)`
146
+ : 'linear-gradient(180deg, rgba(245,236,255,0.06) 0%, rgba(245,236,255,0.03) 100%)',
147
+ border: missed
148
+ ? `0.5px solid color-mix(in oklch, ${accent} 35%, transparent)`
149
+ : `0.5px solid ${CALL.line}`,
150
+ fontFamily: CALL.font,
151
+ color: CALL.ink,
152
+ }}
153
+ >
154
+ {/* Header row β€” icon + title + time */}
155
+ <div
156
+ style={{
157
+ padding: POST_CALL.rowPadding,
158
+ display: 'flex',
159
+ alignItems: 'center',
160
+ gap: 10,
161
+ }}
162
+ >
163
+ <div
164
+ aria-hidden="true"
165
+ style={{
166
+ width: POST_CALL.avatarSize,
167
+ height: POST_CALL.avatarSize,
168
+ borderRadius: '50%',
169
+ background: `color-mix(in oklch, ${accent} 20%, transparent)`,
170
+ border: `0.5px solid color-mix(in oklch, ${accent} 40%, transparent)`,
171
+ display: 'flex',
172
+ alignItems: 'center',
173
+ justifyContent: 'center',
174
+ color: accent,
175
+ flexShrink: 0,
176
+ }}
177
+ >
178
+ {missed ? (
179
+ <IconPhoneMissed size={14} color={accent} />
180
+ ) : (
181
+ <IconPhone size={14} color={accent} />
182
+ )}
183
+ </div>
184
+ <div style={{ flex: 1, minWidth: 0 }}>
185
+ <div
186
+ style={{
187
+ fontSize: 13,
188
+ fontWeight: 600,
189
+ letterSpacing: -0.1,
190
+ color: missed ? accent : CALL.ink,
191
+ whiteSpace: 'nowrap',
192
+ overflow: 'hidden',
193
+ textOverflow: 'ellipsis',
194
+ }}
195
+ >
196
+ {titleText}
197
+ </div>
198
+ <div
199
+ style={{
200
+ fontSize: 11,
201
+ color: CALL.dim,
202
+ fontVariantNumeric: 'tabular-nums',
203
+ }}
204
+ >
205
+ {subtitle}
206
+ </div>
207
+ </div>
208
+ </div>
209
+
210
+ {/* Highlights body β€” italic summary line (only when variant is
211
+ 'highlights' and we're not in the missed state). */}
212
+ {variant === 'highlights' && !missed && summary ? (
213
+ <div
214
+ style={{
215
+ padding: '4px 14px 12px',
216
+ fontFamily: CALL.display,
217
+ fontSize: 15,
218
+ lineHeight: 1.4,
219
+ color: 'rgba(245,236,255,0.82)',
220
+ fontStyle: 'italic',
221
+ }}
222
+ >
223
+ <div
224
+ style={{
225
+ display: 'flex',
226
+ alignItems: 'center',
227
+ gap: 5,
228
+ marginBottom: 4,
229
+ fontFamily: CALL.font,
230
+ fontStyle: 'normal',
231
+ fontSize: 9,
232
+ color: 'oklch(0.80 0.15 340)',
233
+ letterSpacing: 1.4,
234
+ textTransform: 'uppercase',
235
+ }}
236
+ >
237
+ <IconSparkle size={9} color="oklch(0.80 0.15 340)" />
238
+ {personaName} remembers
239
+ </div>
240
+ <span>&ldquo;{summary}&rdquo;</span>
241
+ </div>
242
+ ) : null}
243
+
244
+ {/* Expanded transcript β€” inline, NOT a modal. Renders only for
245
+ variant='expand' when the user has clicked the expand button
246
+ and a transcript was provided. */}
247
+ {variant === 'expand' && expanded && !missed && hasTranscript ? (
248
+ <div
249
+ style={{
250
+ padding: '2px 14px 12px',
251
+ borderTop: `0.5px solid ${CALL.line}`,
252
+ display: 'flex',
253
+ flexDirection: 'column',
254
+ gap: 8,
255
+ }}
256
+ >
257
+ <div
258
+ style={{
259
+ fontSize: 9,
260
+ letterSpacing: 1.4,
261
+ textTransform: 'uppercase',
262
+ color: 'rgba(245,236,255,0.4)',
263
+ paddingTop: 10,
264
+ }}
265
+ >
266
+ Transcript
267
+ </div>
268
+ {(transcript as TranscriptLine[]).map((line, i) => (
269
+ <div key={i} style={{ fontSize: 12, lineHeight: 1.45 }}>
270
+ <span
271
+ style={{
272
+ color:
273
+ line.who === 'user'
274
+ ? 'oklch(0.80 0.15 50)'
275
+ : 'oklch(0.80 0.15 340)',
276
+ fontWeight: 600,
277
+ marginRight: 6,
278
+ }}
279
+ >
280
+ {line.who === 'user' ? 'You' : personaName}
281
+ </span>
282
+ <span style={{ color: 'rgba(245,236,255,0.85)' }}>
283
+ {line.text}
284
+ </span>
285
+ </div>
286
+ ))}
287
+ </div>
288
+ ) : null}
289
+
290
+ {/* CTA row */}
291
+ <div
292
+ style={{
293
+ display: 'flex',
294
+ gap: 6,
295
+ padding: '8px 10px 10px',
296
+ borderTop:
297
+ variant === 'expand' && expanded && !missed && hasTranscript
298
+ ? 'none'
299
+ : `0.5px solid ${CALL.line}`,
300
+ }}
301
+ >
302
+ <button
303
+ type="button"
304
+ onClick={missed ? (onCallBack ?? onResume) : onResume}
305
+ disabled={!(missed ? (onCallBack ?? onResume) : onResume)}
306
+ aria-label={missed ? 'Call back' : 'Resume call'}
307
+ style={{
308
+ flex: 1,
309
+ height: POST_CALL.ctaHeight,
310
+ borderRadius: POST_CALL.ctaRadius,
311
+ cursor: 'pointer',
312
+ background: accent,
313
+ color: '#0c0810',
314
+ border: 'none',
315
+ fontFamily: CALL.font,
316
+ fontSize: 12,
317
+ fontWeight: 600,
318
+ display: 'flex',
319
+ alignItems: 'center',
320
+ justifyContent: 'center',
321
+ gap: 6,
322
+ }}
323
+ >
324
+ <IconPhone size={12} color="#0c0810" />
325
+ {missed ? 'Call back' : 'Resume call'}
326
+ </button>
327
+
328
+ {/* Secondary CTA β€” only the non-missed variants have one */}
329
+ {!missed && variant === 'highlights' && onOpenFullTranscript ? (
330
+ <button
331
+ type="button"
332
+ onClick={onOpenFullTranscript}
333
+ aria-label="Open full transcript"
334
+ style={{
335
+ flex: 1,
336
+ height: POST_CALL.ctaHeight,
337
+ borderRadius: POST_CALL.ctaRadius,
338
+ cursor: 'pointer',
339
+ background: 'rgba(245,236,255,0.06)',
340
+ color: CALL.ink,
341
+ border: `0.5px solid ${CALL.line}`,
342
+ fontFamily: CALL.font,
343
+ fontSize: 12,
344
+ fontWeight: 500,
345
+ }}
346
+ >
347
+ Full transcript
348
+ </button>
349
+ ) : null}
350
+
351
+ {!missed && variant === 'expand' && hasTranscript ? (
352
+ <button
353
+ type="button"
354
+ onClick={() => setExpanded((v) => !v)}
355
+ aria-expanded={expanded}
356
+ aria-label={expanded ? 'Collapse transcript' : 'View transcript'}
357
+ style={{
358
+ flex: 1,
359
+ height: POST_CALL.ctaHeight,
360
+ borderRadius: POST_CALL.ctaRadius,
361
+ cursor: 'pointer',
362
+ background: 'rgba(245,236,255,0.06)',
363
+ color: CALL.ink,
364
+ border: `0.5px solid ${CALL.line}`,
365
+ fontFamily: CALL.font,
366
+ fontSize: 12,
367
+ fontWeight: 500,
368
+ }}
369
+ >
370
+ {expanded ? 'Collapse β–΄' : 'View transcript β–Ύ'}
371
+ </button>
372
+ ) : null}
373
+ </div>
374
+ </div>
375
+ )
376
+ }
377
+
378
+ export default PostCallCard
379
+
380
+ // Test hooks β€” exposed for unit tests without widening the public API.
381
+ export const postCallCardInternals = {
382
+ formatDuration,
383
+ formatClock,
384
+ }
frontend/src/ui/phone/README.md ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # `ui/phone/` β€” call surface component library
2
+
3
+ Status: **partial port** of the design system from the engineering
4
+ handoff (`Phone Call UI.html`). This README tracks what ships today
5
+ and what's queued for follow-up sessions.
6
+
7
+ ## What's here now
8
+
9
+ ```
10
+ phone/
11
+ β”œβ”€β”€ README.md this file
12
+ β”œβ”€β”€ tokens.ts CALL palette + POST_CALL layout constants
13
+ β”œβ”€β”€ icons.tsx shared SVG icon set (mic, phone*, chat, …)
14
+ β”œβ”€β”€ PostCallCard.tsx inline chat card, expandable transcript,
15
+ β”‚ 3 variants (expand / highlights / missed)
16
+ β”œβ”€β”€ PostCallCard.test.tsx 13 tests β€” variant matrix + keyboard/click
17
+ └── primitives/
18
+ β”œβ”€β”€ index.ts barrel β€” single import path
19
+ β”œβ”€β”€ useReducedMotion.ts OS motion-preference subscription
20
+ β”œβ”€β”€ useReducedMotion.test.ts 4 tests
21
+ β”œβ”€β”€ useFocusTrap.ts keyboard-focus containment for dialogs
22
+ β”œβ”€β”€ useFocusTrap.test.tsx 6 tests
23
+ β”œβ”€β”€ ControlBtn.tsx circular action button
24
+ β”œβ”€β”€ ControlBtn.test.tsx 8 tests
25
+ β”œβ”€β”€ Waveform.tsx rAF-driven audio-level bars
26
+ β”œβ”€β”€ Waveform.test.tsx 9 tests
27
+ β”œβ”€β”€ Aura.tsx seeded-hue persona identity chip
28
+ β”œβ”€β”€ Aura.test.tsx 8 tests
29
+ β”œβ”€β”€ AmbientAura.tsx page-scale coloured backdrop glow
30
+ └── AmbientAura.test.tsx 7 tests
31
+ ```
32
+
33
+ Consumed by:
34
+
35
+ - `frontend/src/ui/App.tsx` β€” renders `PostCallCard` inline in the
36
+ chat stream for every `Msg` that carries a `callMemory` payload.
37
+ The card is **not** a modal β€” it sits in the conversation so the
38
+ user can expand the transcript without a context switch.
39
+ - `primitives/` are additive foundation for the next batch
40
+ (Waveform, Aura, AmbientAura, CallScreen). No consumers yet;
41
+ `CallOverlay` still has its own inlined versions of waveform +
42
+ avatar + control buttons, which the next session extracts.
43
+
44
+ ## What's queued for follow-up sessions
45
+
46
+ The handoff defines a much larger set of components. They are
47
+ additive, so shipping them one at a time doesn't block the current
48
+ call feature β€” it's already functional end-to-end with the
49
+ `CallOverlay` + the inline `PostCallCard`.
50
+
51
+ | File | Status | Notes |
52
+ |---|---|---|
53
+ | `primitives/ControlBtn.tsx` | **Shipped** | 8 tests. CallOverlay still inlines its own; extraction queued behind CallScreen. |
54
+ | `primitives/useReducedMotion.ts` | **Shipped** | 4 tests. Drives the motion gate on Waveform + Aura + AmbientAura. |
55
+ | `primitives/useFocusTrap.ts` | **Shipped** | 6 tests. Consumed by CallScreen + LockScreenIncoming when they land. |
56
+ | `primitives/Waveform.tsx` | **Shipped + extracted** | 9 tests. CallOverlay's inline CallWaveform is gone β€” `<Waveform mode={waveformModeFromCallState(state)} intensityRef={…} />` takes its place. |
57
+ | `primitives/Aura.tsx` | **Shipped + extracted** | 8 tests. FNV-1a hashed seeded hue + optional photoUrl path. Inner disc of CallOverlay's CallAvatar now delegates to `<Aura />`; the halo + pulse rings stay outside (state-dependent). |
58
+ | `primitives/AmbientAura.tsx` | **Shipped** | 7 tests. Additive β€” CallOverlay today uses a flat dim+blur backdrop; this primitive is slated for CallScreen's fullscreen composition. |
59
+ | `CallScreen.tsx` | Pending | Fullscreen mobile composition. Desktop uses the existing `CallOverlay` modal. |
60
+ | `CallOverlay.tsx` (refactor) | Pending | Route between fullscreen / modal / PiP based on viewport. Today's `CallOverlay.tsx` is desktop-modal-only. |
61
+ | `TextInCall.tsx` | Pending | Mid-call composer sheet β€” minimize call to a thin bar, slide composer up, send text over the existing chat REST. |
62
+ | `HangUpTransition.tsx` | Pending | ~1 s "call ended" fade beat before the overlay dismisses. Today the overlay fades inline. |
63
+ | `MutedToast.tsx` | Pending | Brief "you're muted" confirmation pill. Today the muted state is indicated only by the waveform colour. |
64
+ | `LockScreenIncoming.tsx` | Pending | iOS-style full-bleed incoming with slide-to-answer. PWA only. |
65
+ | `PoorConnection.tsx` | Pending | Reconnecting banner when WS has been silent >3 s. Today the socket reconnects silently. |
66
+
67
+ ## Suggested sequencing for the next session
68
+
69
+ Small commits, one per component file, matching the pattern from
70
+ this session:
71
+
72
+ 1. `primitives/Waveform.tsx` β€” extract the `rAF`-driven version
73
+ from `CallOverlay.tsx`, swap `CallOverlay`'s inline copy for
74
+ the import.
75
+ 2. `primitives/Aura.tsx` β€” extract `CallAvatar`, thread through
76
+ `CallOverlay` + `PostCallCard` so the avatar identity is
77
+ consistent across surfaces.
78
+ 3. `primitives/ControlBtn.tsx` β€” same.
79
+ 4. `CallScreen.tsx` β€” fullscreen mobile composition on top of the
80
+ primitives.
81
+ 5. `HangUpTransition.tsx` + `MutedToast.tsx` β€” small polish
82
+ additions; each a single new render branch in `CallOverlay`.
83
+ 6. `TextInCall.tsx` β€” biggest piece, needs its own focus session.
84
+ 7. `LockScreenIncoming.tsx` β€” PWA-only, coordinates with service-
85
+ worker push; scope-separate.
86
+ 8. `PoorConnection.tsx` β€” layer on `callSocket`'s existing silence
87
+ watchdog.
88
+
89
+ Keep each batch under ~300 LOC and write a vitest for every
90
+ component (the 13-test PostCallCard file is the template).
91
+
92
+ ## Design reference
93
+
94
+ The pixel-level source of truth is `Phone Call UI.html` in the
95
+ design project β€” matches every state and surface. Each TSX file in
96
+ this folder maps 1:1 to a named component in the design canvas.
97
+
98
+ ## Not in scope (anywhere in this folder)
99
+
100
+ - New backend endpoints
101
+ - Replacing the `voice_call` module
102
+ - Replacing `CallOverlay`'s streaming / barge-in plumbing
103
+ - Group calls / add participant
frontend/src/ui/phone/icons.tsx ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /**
2
+ * phone/icons.tsx β€” shared SVG icon set for the call surface.
3
+ *
4
+ * Direct-return functions (no shared base helper) so each export is
5
+ * trivially tree-shakable and the module's export table stays
6
+ * predictable under the vitest transform (a helper-based variant
7
+ * earlier in this branch caused every export to vanish from the
8
+ * module namespace β€” the root cause wasn't isolated, but the
9
+ * direct-return form sidesteps it entirely).
10
+ *
11
+ * Stroke weight is 1.6 for mic/chat/speaker/video (secondary
12
+ * actions) and 1.8 for phone icons + back (primary + nav) to match
13
+ * the design spec in ``Phone Call UI.html``. Icons inherit
14
+ * currentColor by default so a parent text colour change cascades
15
+ * cleanly.
16
+ */
17
+
18
+ import React from 'react'
19
+
20
+ interface IconProps {
21
+ size?: number
22
+ color?: string
23
+ title?: string
24
+ }
25
+
26
+ type SvgExtras = {
27
+ role: 'img' | 'presentation'
28
+ 'aria-hidden'?: 'true'
29
+ 'aria-label'?: string
30
+ }
31
+
32
+ function a11y(title?: string): SvgExtras {
33
+ return title
34
+ ? { role: 'img', 'aria-label': title }
35
+ : { role: 'presentation', 'aria-hidden': 'true' }
36
+ }
37
+
38
+ // ── Icons ─────────────────────────────────────────────────────────
39
+
40
+ export const IconMic: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
41
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
42
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
43
+ {...a11y(title)}>
44
+ {title ? <title>{title}</title> : null}
45
+ <rect x="9" y="3" width="6" height="12" rx="3" />
46
+ <path d="M5 11a7 7 0 0 0 14 0M12 18v3" />
47
+ </svg>
48
+ )
49
+
50
+ export const IconMicOff: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
51
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
52
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
53
+ {...a11y(title)}>
54
+ {title ? <title>{title}</title> : null}
55
+ <path d="M9 9V6a3 3 0 0 1 6 0v5m0 4a3 3 0 0 1-6 0" />
56
+ <path d="M5 11a7 7 0 0 0 11.5 5.3M19 11a7 7 0 0 1-.4 2.3M12 18v3" />
57
+ <path d="M3 3l18 18" />
58
+ </svg>
59
+ )
60
+
61
+ export const IconPhone: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
62
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
63
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
64
+ {...a11y(title)}>
65
+ {title ? <title>{title}</title> : null}
66
+ <path d="M5 4h3l2 5-2.5 1.5a11 11 0 0 0 6 6L15 14l5 2v3a2 2 0 0 1-2 2A16 16 0 0 1 3 6a2 2 0 0 1 2-2z" />
67
+ </svg>
68
+ )
69
+
70
+ export const IconPhoneEnd: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
71
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
72
+ stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
73
+ {...a11y(title)}>
74
+ {title ? <title>{title}</title> : null}
75
+ <path d="M4 14c5-5 11-5 16 0l-2 2-3-1v-2a9 9 0 0 0-6 0v2l-3 1-2-2z"
76
+ transform="rotate(135 12 12)" />
77
+ </svg>
78
+ )
79
+
80
+ /** Phone with superimposed slash β€” used by PostCallCard's missed
81
+ * variant so the missed state reads differently from a normal
82
+ * call event even at thumbnail sizes. */
83
+ export const IconPhoneMissed: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
84
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
85
+ stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
86
+ {...a11y(title)}>
87
+ {title ? <title>{title}</title> : null}
88
+ <path d="M22 16.92v3a2 2 0 01-2.18 2 19.79 19.79 0 01-8.63-3.07 19.5 19.5 0 01-6-6 19.79 19.79 0 01-3.07-8.67A2 2 0 014.11 2h3a2 2 0 012 1.72 12.84 12.84 0 00.7 2.81 2 2 0 01-.45 2.11L8.09 9.91a16 16 0 006 6l1.27-1.27a2 2 0 012.11-.45 12.84 12.84 0 002.81.7 2 2 0 011.72 2.03z" />
89
+ <line x1="23" y1="1" x2="17" y2="7" />
90
+ <line x1="17" y1="1" x2="23" y2="7" />
91
+ </svg>
92
+ )
93
+
94
+ export const IconChat: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
95
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
96
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
97
+ {...a11y(title)}>
98
+ {title ? <title>{title}</title> : null}
99
+ <path d="M4 5h16v11H9l-5 4z" />
100
+ <circle cx="9" cy="10.5" r="0.9" fill={color} stroke="none" />
101
+ <circle cx="12" cy="10.5" r="0.9" fill={color} stroke="none" />
102
+ <circle cx="15" cy="10.5" r="0.9" fill={color} stroke="none" />
103
+ </svg>
104
+ )
105
+
106
+ export const IconSpeaker: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
107
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
108
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
109
+ {...a11y(title)}>
110
+ {title ? <title>{title}</title> : null}
111
+ <path d="M4 9v6h4l5 4V5L8 9H4z" />
112
+ <path d="M17 8a5 5 0 0 1 0 8M19.5 5.5a8.5 8.5 0 0 1 0 13" />
113
+ </svg>
114
+ )
115
+
116
+ export const IconVideo: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
117
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
118
+ stroke={color} strokeWidth="1.6" strokeLinecap="round" strokeLinejoin="round"
119
+ {...a11y(title)}>
120
+ {title ? <title>{title}</title> : null}
121
+ <rect x="3" y="6" width="13" height="12" rx="2" />
122
+ <path d="M16 10l5-3v10l-5-3z" />
123
+ </svg>
124
+ )
125
+
126
+ export const IconBack: React.FC<IconProps> = ({ size = 22, color = 'currentColor', title }) => (
127
+ <svg width={size} height={size} viewBox="0 0 24 24" fill="none"
128
+ stroke={color} strokeWidth="1.8" strokeLinecap="round" strokeLinejoin="round"
129
+ {...a11y(title)}>
130
+ {title ? <title>{title}</title> : null}
131
+ <path d="M15 5l-7 7 7 7" />
132
+ </svg>
133
+ )
134
+
135
+ /** Small star glyph used by the "persona remembers" label on the
136
+ * highlights variant of PostCallCard. Fill-mode render (not stroke)
137
+ * reads better at 9–11 px. */
138
+ export const IconSparkle: React.FC<IconProps> = ({ size = 12, color = 'currentColor', title }) => (
139
+ <svg width={size} height={size} viewBox="0 0 24 24" fill={color}
140
+ {...a11y(title)}>
141
+ {title ? <title>{title}</title> : null}
142
+ <path d="M12 2l1.8 6.2L20 10l-6.2 1.8L12 18l-1.8-6.2L4 10l6.2-1.8z" />
143
+ </svg>
144
+ )