diff --git a/.gitattributes b/.gitattributes index 69b421caa00aa98567b55e83e187c08b6e889c2f..827a1d818417e2cd6d39528abc34a5497375d3ae 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,9 +1,9 @@ -*.jpeg filter=lfs diff=lfs merge=lfs -text -*.woff filter=lfs diff=lfs merge=lfs -text -*.ico filter=lfs diff=lfs merge=lfs -text *.hpersona filter=lfs diff=lfs merge=lfs -text -*.png filter=lfs diff=lfs merge=lfs -text *.jpg filter=lfs diff=lfs merge=lfs -text -*.webp filter=lfs diff=lfs merge=lfs -text *.svg filter=lfs diff=lfs merge=lfs -text *.woff2 filter=lfs diff=lfs merge=lfs -text +*.ico filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpeg filter=lfs diff=lfs merge=lfs -text +*.webp filter=lfs diff=lfs merge=lfs -text +*.woff filter=lfs diff=lfs merge=lfs -text diff --git a/backend/app/main.py b/backend/app/main.py index f82e1019e9c62df1524d90a89c0a2b43617ad200..79670ca5b5d152f8acc7129034eff8c67ed7d23f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -159,6 +159,44 @@ app.include_router(teams_router) from .teams.bridge_routes import router as teams_bridge_router app.include_router(teams_bridge_router) +# --- Voice call (/v1/voice-call/*) — ADDITIVE, FEATURE-FLAGGED --------------- +# Fully off by default. Enable with ``VOICE_CALL_ENABLED=true``. If anything +# in the voice_call module raises on import (bad migration, missing dep, …) +# we swallow it so the rest of the app stays up — this feature is optional +# and must never take production down. +try: + from .voice_call.config import load as _voice_call_load_config + _voice_call_cfg = _voice_call_load_config() + if _voice_call_cfg.enabled: + from .voice_call.router import build_router as _build_vc_router + from .voice_call.router import build_ws_router as _build_vc_ws_router + app.include_router(_build_vc_router(_voice_call_cfg)) + app.include_router(_build_vc_ws_router(_voice_call_cfg)) + print("[voice_call] enabled — routes mounted under /v1/voice-call") +except Exception as _vc_err: # noqa: BLE001 + # Deliberate broad catch: this entire feature is optional. Log and move on. + print(f"[voice_call] DISABLED due to import error: {_vc_err}") + +# --- Persona call (/v1/persona-call/*) — ADDITIVE, FEATURE-FLAGGED ----------- +# Phone-call conversational dynamics + per-turn system suffix composer. +# Off by default. Enable with ``PERSONA_CALL_ENABLED=true``. The module +# operates behind voice_call/ws.py — the HTTP router is minor (facets +# lookup, state inspection, shadow-directive audit). Independent flag so +# it can ship in shadow-compose mode (``PERSONA_CALL_APPLY=false``) +# before any prompt surface actually changes. +try: + from .persona_call.config import load as _persona_call_load_config + _persona_call_cfg = _persona_call_load_config() + if _persona_call_cfg.enabled: + from .persona_call.router import build_router as _build_pc_router + app.include_router(_build_pc_router(_persona_call_cfg)) + print( + f"[persona_call] enabled (apply={_persona_call_cfg.apply}) — " + "routes mounted under /v1/persona-call" + ) +except Exception as _pc_err: # noqa: BLE001 + print(f"[persona_call] DISABLED due to import error: {_pc_err}") + # Include Marketplace routes (/v1/marketplace/*) app.include_router(marketplace_router) diff --git a/backend/app/persona_call/__init__.py b/backend/app/persona_call/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ea5e1d9910b2a09d8f7eec52ff57cad770b0e063 --- /dev/null +++ b/backend/app/persona_call/__init__.py @@ -0,0 +1,33 @@ +""" +persona_call — phone-call conversational dynamics on top of voice_call. + +Adds nothing to a persona's BASE system prompt. Instead, when a persona +is on a live phone call, this module composes a small per-turn system +*suffix* that: + + * tells the persona it is currently on a phone call (not chat), + * applies one research-grounded rule from the design doc (e.g. + "skip 'how are you' after 9 pm local"), + * enforces the two-phase closing handshake at the state machine level + (the only place the model is overridden rather than advised), + * emits server-originated *backchannels* and *fillers* on a separate + low-volume stream so the user hears continuers + thinking sounds + without waiting for a full LLM turn. + +Fully additive + feature-flagged. With ``PERSONA_CALL_ENABLED=false`` +(the default) this module is never imported and no row is ever written +to the ``persona_call_state`` table. + +Published research baked in: + Schegloff 1968, Schegloff & Sacks 1973, Sacks/Schegloff/Jefferson 1974, + Jefferson 1984, Stivers et al. 2009, Clark & French 1981, + Local & Walker 2012, OpenAI Realtime Prompting Guide, Sesame 2025. + +Call-centre software-engineering practice applied: + single-source-of-truth state machine; every behavior has a named + module and a test anchor; deterministic telemetry counters for + naturalness SLOs; zero coupling to persona storage. +""" +from __future__ import annotations + +__all__ = ["config", "directive", "router"] diff --git a/backend/app/persona_call/backchannel.py b/backend/app/persona_call/backchannel.py new file mode 100644 index 0000000000000000000000000000000000000000..70f294e55e9e26614a9c0f8f1c53c1466f8e5320 --- /dev/null +++ b/backend/app/persona_call/backchannel.py @@ -0,0 +1,122 @@ +""" +Server-originated backchannel emission — the separate low-volume +stream (per the user's chosen default). + +A backchannel is NOT a turn. It's a "mm-hm" / "yeah" / "got it" the +server emits *while the user is still speaking* so the caller hears +recipiency instead of silence. Research (Jefferson 1984) shows humans +emit one every 2–3 clauses on the phone — they compensate for the +absence of visual nods. + +Design rules baked in: + + * Token pulled from the persona's own pool (VoiceFacets.ack_tokens) + so Darkangel666 sounds like Darkangel666 even when backchanneling. + * Cadence: at most one per ``backchannel_min_gap_ms``, and only after + ``backchannel_clause_trigger`` clause boundaries in the CURRENT + user utterance. + * Anti-repetition: never the same token two backchannels in a row. + * Played at ``backchannel_volume_db`` below the main TTS stream so + it overlaps the user's voice without fighting it (the client + applies the gain — this module just names the dB). + * Skipped entirely when ``facets.backchannel_rate`` is 0. + +The emission itself is a WS event; this module picks the token and +decides IF/WHEN. The actual ``ws.send`` call lives in ws.py. +""" +from __future__ import annotations + +import random +import re +import time +from typing import Any, Dict, List, Optional, Tuple + +from .config import PersonaCallConfig +from .facets import VoiceFacets + + +# Rough clause boundaries — commas, semicolons, sentence enders, +# conjunctions that start a clause ("and then…", "but…", "so…"). +# Runs on the partial STT transcript, so it fires as the user speaks. +_CLAUSE_BOUNDARY = re.compile( + r"(" + r"[,;]|[.!?]\s+|" # punctuation + r"\b(and|but|so|because|then|however|actually)\b\s" # discourse markers + r")", + re.I, +) + + +def count_clause_boundaries(partial_text: str) -> int: + """How many clause boundaries have been uttered in the current + user turn so far.""" + if not partial_text: + return 0 + return len(_CLAUSE_BOUNDARY.findall(partial_text)) + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def choose_token( + facets: VoiceFacets, + *, + recent_acks: List[str], + seed_utterance: str = "", +) -> Optional[str]: + """Pick a backchannel token that is (a) in the persona's pool and + (b) not the most recent one. Returns None iff the pool is empty. + + Mild stance-sensitivity: if the partial utterance ended with a + rising question mark, we bias toward stance tokens ('oh really?', + 'yeah?'); falling-period endings bias toward acknowledgments. + """ + pool = [t for t in facets.ack_tokens if t] + if not pool: + return None + last = (recent_acks[-1].lower() if recent_acks else "") + candidates = [t for t in pool if t.lower() != last] or pool + # Stance nudge — prefer single-syllable quick tokens on + # exclamations / questions, longer ones on plain pauses. + is_query = bool(seed_utterance.rstrip().endswith(("?", "!"))) + if is_query: + candidates = sorted(candidates, key=lambda t: len(t)) + return random.choice(candidates) + + +def should_emit( + *, + partial_text: str, + last_backchannel_ms: int, + facets: VoiceFacets, + cfg: PersonaCallConfig, +) -> bool: + """True iff we should send a backchannel right now. Pure decision; + no side effects.""" + if facets.backchannel_rate <= 0: + return False + # Scale the clause threshold by the persona's own rate. Low-rate + # personas (e.g. reserved / clinical) need more clauses before a + # backchannel; chatty personas need fewer. + trigger = max( + 1, int(round(cfg.backchannel_clause_trigger / max(0.25, facets.backchannel_rate))) + ) + if count_clause_boundaries(partial_text) < trigger: + return False + min_gap = int(cfg.backchannel_min_gap_ms / max(0.25, facets.backchannel_rate)) + if _now_ms() - int(last_backchannel_ms or 0) < min_gap: + return False + return True + + +def build_event(token: str, volume_db: float) -> Dict[str, Any]: + """Shape the WS event payload ws.py will serialize.""" + return { + "type": "assistant.backchannel", + "payload": { + "token": token, + "volume_db": float(volume_db), + "ts": _now_ms(), + }, + } diff --git a/backend/app/persona_call/closing.py b/backend/app/persona_call/closing.py new file mode 100644 index 0000000000000000000000000000000000000000..e941817673b817e1ac0d761d650e2422c44c7f2b --- /dev/null +++ b/backend/app/persona_call/closing.py @@ -0,0 +1,102 @@ +""" +Two-phase closing handshake — the one place we *enforce* instead of +advise. + +Why enforced: LLMs reliably collapse Schegloff's 3–6-turn closing +into "Goodbye!" the moment they hear an "ok". Humans don't. The +handshake is a short, mandatory state transition that keeps the +persona from hanging up before the caller does. + +Flow (per Schegloff & Sacks 1973, *Opening up Closings*): + + user says "ok" / "alright" / "I should go" / "thanks, that's all" + │ + ▼ + state.phase := 'pre_closing' + pre_closing_trigger := + │ + ▼ + persona reply directive FORCES a pre-closing token reply: + - one short softener from facets.pre_closing_tokens + - no "bye", no "goodbye", no "take care" yet + │ + ▼ (next user turn) + if user says "bye" / "talk later" / "see ya": + persona reply directive FORCES a mirrored two-part close: + , + e.g. "take care, bye" + else: + state stays in 'pre_closing'; persona speaks a normal turn. +""" +from __future__ import annotations + +import re +from typing import Optional + +from .facets import VoiceFacets + + +# User utterances that start a closing (pre-closing trigger). +# Kept tight to avoid false positives mid-conversation. +_PRE_CLOSE_PATTERNS = [ + re.compile(p, re.I) for p in [ + r"^\s*(ok|okay|alright|ok\s+then)\s*[.!?]?\s*$", + r"\b(i should (go|run|get back)|gotta (go|run)|need to go)\b", + r"\b(thanks[, ]+(that'?s|that is)\s+(all|it))\b", + r"\b(talk (to you )?(later|soon)|call (you )?(later|back))\b", + ] +] + +# Pure terminal — "bye"-family tokens that complete the exchange. +_TERMINAL_PATTERNS = [ + re.compile(p, re.I) for p in [ + r"^\s*(bye|good[\- ]?bye|later|see ya|see you|ciao)\s*[.!?]?\s*$", + r"\b(bye bye|byebye)\b", + ] +] + + +def is_pre_closing_trigger(user_text: str) -> bool: + """True iff the user just began a closing sequence.""" + if not user_text: + return False + return any(p.search(user_text) for p in _PRE_CLOSE_PATTERNS) + + +def is_terminal(user_text: str) -> bool: + """True iff the user said goodbye — the second half of the + two-phase close. Matched INDEPENDENTLY of the pre-close trigger + so the state machine can skip straight to terminal if the user + blurts out "bye!" without a prior "ok".""" + if not user_text: + return False + return any(p.search(user_text) for p in _TERMINAL_PATTERNS) + + +# ── directive fragments for each phase ─────────────────────────────── + +def pre_closing_directive(facets: VoiceFacets, trigger: Optional[str]) -> str: + """Directive the composer injects when we're in 'pre_closing' + but the user hasn't said 'bye' yet.""" + pool = ", ".join(facets.pre_closing_tokens[:4]) + trig = f'"{trigger}"' if trigger else "a closing cue" + return ( + f"The caller offered a pre-closing ({trig}). " + "Reciprocate with ONE matching soft pre-closing token — " + f"something from: {pool}. Do NOT say 'goodbye' yet. " + "Keep the turn to ≤ 7 words and leave the actual goodbye to " + "the caller. Research: Schegloff & Sacks 1973 — the terminal " + "exchange is the caller's to initiate." + ) + + +def terminal_directive(facets: VoiceFacets) -> str: + """Directive for the actual bye-for-bye turn.""" + softeners = ", ".join(facets.pre_closing_tokens[:3]) + closings = ", ".join(facets.closing_tokens[:3]) + return ( + "The caller just said goodbye. Mirror them with a TWO-PART " + f"close: one softener (e.g. {softeners}) followed by one " + f"terminal (e.g. {closings}). Never a single bare 'goodbye'. " + "Keep it to ≤ 6 words. After this turn the call ends." + ) diff --git a/backend/app/persona_call/config.py b/backend/app/persona_call/config.py new file mode 100644 index 0000000000000000000000000000000000000000..340c621d38b034f6ce7894a3a13e230760cd5204 --- /dev/null +++ b/backend/app/persona_call/config.py @@ -0,0 +1,102 @@ +""" +Feature flags + runtime knobs for persona_call. + +Every knob is an env var with a safe default. The module ships off. + +Two flags, not one, so "shadow compose" (§ rollout P1 in the design) +can run without changing any outgoing prompt: + + PERSONA_CALL_ENABLED — master. When false, this module is never + loaded into the app. + PERSONA_CALL_APPLY — when ENABLED=true but APPLY=false, the + composer runs and its output is logged to + ``persona_call_directives`` but NOT injected + as a system message. Lets us collect + production directives for a week before + letting them shape turns. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in ("1", "true", "yes", "on") + + +def _int_env(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except (TypeError, ValueError): + return default + + +def _float_env(name: str, default: float) -> float: + raw = os.getenv(name) + if raw is None: + return default + try: + return float(raw) + except (TypeError, ValueError): + return default + + +@dataclass(frozen=True) +class PersonaCallConfig: + """Resolved at every :func:`load` call — cheap, keep fresh for tests.""" + + # Master flag. False = this module is not imported into main.py. + enabled: bool + # When True, the composed directive is actually sent to the model. + # When False, it's logged only (shadow mode). + apply: bool + + # Backchannel stream thresholds. + backchannel_clause_trigger: int # emit after N clause boundaries + backchannel_min_gap_ms: int # don't emit more often than this + backchannel_volume_db: float # played at -6 dB by default + + # Filler emission during tool/LLM wait (Stivers 700 ms trouble line). + filler_emit_after_ms: int # default 600 + + # Phase machine thresholds. + how_are_you_late_hour: int # skip "how are you" at or after + late_night_brevity_hour: int # enter brevity bias at or after + late_night_brevity_hour_end: int # exit brevity bias at or before + + # Anti-repetition ledger sizes. + ack_ledger_window: int + opener_ledger_window: int + + # Closing handshake — the one enforced override. + closing_trigger_tokens_extra: str # comma-separated, user-supplied + + # Reason-for-call fallback turn. + reason_fallback_turn: int # at this turn, ask if nothing heard + + +def load() -> PersonaCallConfig: + return PersonaCallConfig( + enabled=_bool_env("PERSONA_CALL_ENABLED", False), + apply=_bool_env("PERSONA_CALL_APPLY", True), + backchannel_clause_trigger=_int_env("PERSONA_CALL_BC_CLAUSES", 2), + backchannel_min_gap_ms=_int_env("PERSONA_CALL_BC_MIN_GAP_MS", 1800), + backchannel_volume_db=_float_env("PERSONA_CALL_BC_VOL_DB", -6.0), + filler_emit_after_ms=_int_env("PERSONA_CALL_FILLER_AFTER_MS", 600), + how_are_you_late_hour=_int_env("PERSONA_CALL_HAY_LATE_HOUR", 21), + late_night_brevity_hour=_int_env("PERSONA_CALL_LATE_HOUR", 22), + late_night_brevity_hour_end=_int_env("PERSONA_CALL_LATE_HOUR_END", 6), + ack_ledger_window=_int_env("PERSONA_CALL_ACK_WINDOW", 5), + opener_ledger_window=_int_env("PERSONA_CALL_OPENER_WINDOW", 3), + closing_trigger_tokens_extra=os.getenv( + "PERSONA_CALL_CLOSING_EXTRA_TOKENS", "" + ), + reason_fallback_turn=_int_env("PERSONA_CALL_REASON_FALLBACK_TURN", 3), + ) diff --git a/backend/app/persona_call/context.py b/backend/app/persona_call/context.py new file mode 100644 index 0000000000000000000000000000000000000000..a706e55436aa00421c0e1d5c8a74e93131b63f8f --- /dev/null +++ b/backend/app/persona_call/context.py @@ -0,0 +1,191 @@ +""" +Environmental + caller-context inputs for the per-turn composer. + +Three kinds of signal, all computed on demand (never stored): + + (a) Local clock — hour, day-of-week, weekend bit. Resolved from the + caller's ``device_info.tz`` when present, else server tz, else UTC. + Late-night brevity and morning-energy rules key off this. + + (b) Recency — weeks since this (user, persona) last spoke. Drives the + "hey, it's been a while" one-shot acknowledgment. + + (c) Caller-state extraction from STT — a tiny regex/keyword classifier + that runs on every user utterance: + * mobility (driving / walking / on transit) + * time pressure (quick question, running late, stepping out) + * reason-for-call sentence (turn ≤ 3) + +All inference is local + deterministic. No LLM calls in this module. +""" +from __future__ import annotations + +import datetime as _dt +import re +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + + +# ── local clock ─────────────────────────────────────────────────────── + +@dataclass(frozen=True) +class CallEnv: + local_hour: int + day_of_week: int # 0 Mon … 6 Sun + is_weekend: bool + weeks_since_last_call: int # -1 when never + had_unfinished_topic: bool + resumed_from_topic: Optional[str] = None + + +def _resolve_tz(tz_name: Optional[str]): + """Best-effort tz resolution. Returns a tzinfo or UTC.""" + if not tz_name: + return _dt.timezone.utc + try: + # Python 3.9+ stdlib + from zoneinfo import ZoneInfo + return ZoneInfo(tz_name) + except Exception: + return _dt.timezone.utc + + +def compute_env( + *, + tz: Optional[str] = None, + weeks_since_last_call: int = -1, + resumed_from_topic: Optional[str] = None, + now: Optional[_dt.datetime] = None, +) -> CallEnv: + """Build a CallEnv from primitive inputs. No DB access here — + ws.py does the session lookups and calls this with the results so + this function stays cheap and unit-testable.""" + tzinfo = _resolve_tz(tz) + current = (now or _dt.datetime.now(tz=_dt.timezone.utc)).astimezone(tzinfo) + return CallEnv( + local_hour=current.hour, + day_of_week=current.weekday(), + is_weekend=current.weekday() >= 5, + weeks_since_last_call=weeks_since_last_call, + had_unfinished_topic=bool(resumed_from_topic), + resumed_from_topic=resumed_from_topic, + ) + + +# ── caller-state extraction ─────────────────────────────────────────── + +# Deliberately narrow regexes. False positives in this module produce +# a directive like "the caller is driving — keep it to one sentence", +# which is always a safe fallback; false negatives lose the rule, which +# is also safe. So tight > comprehensive. + +_MOBILITY_DRIVING = re.compile( + r"\b(i[' ]?m|i am|currently|just)?\s*(driving|behind the wheel|on the road|in the car)\b", + re.I, +) +_MOBILITY_WALKING = re.compile( + r"\b(i[' ]?m|i am)?\s*(walking|out on a walk|going for a walk)\b", + re.I, +) +_MOBILITY_TRANSIT = re.compile( + r"\b(on (the )?(train|subway|tube|metro|bus)|at the (station|airport)|boarding)\b", + re.I, +) +_TIME_PRESSURE = re.compile( + r"\b(" + r"quick question|only have a minute|short on time|running late|" + r"stepping out|have to run|in a rush|in a hurry|gotta be quick" + r")\b", + re.I, +) + +# Very loose reason-for-call cues for turn ≤ 3. We don't need to +# extract a perfect paraphrase — we just need a yes/no flag so the +# directive composer can decide whether the phase machine can move +# out of 'opening'. +# +# Intentionally does NOT match "quick question" — that is a +# time-pressure signal, not an articulated reason. The composer reads +# both signals and will still skip 'how are you' on time pressure +# alone; we just don't want to prematurely promote phase → topic +# before the skip rule fires. +_REASON_CUE = re.compile( + r"\b(" + r"i('?m| am)?\s+calling (about|to|because|for)|" + r"i wanted to (talk|ask|check|discuss|tell)|" + r"the reason i('?m| am) calling|" + r"i need (to know|help with|your help|a hand)" + r")\b", + re.I, +) + + +@dataclass +class CallerSignal: + driving: bool = False + walking: bool = False + transit: bool = False + time_pressured: bool = False + reason_expressed: bool = False + reason_text: str = "" + + def merge(self, other: "CallerSignal") -> "CallerSignal": + """Accumulate across multiple user utterances in one call. + Once a flag is set, it stays set until the phase machine + resets it.""" + return CallerSignal( + driving=self.driving or other.driving, + walking=self.walking or other.walking, + transit=self.transit or other.transit, + time_pressured=self.time_pressured or other.time_pressured, + reason_expressed=self.reason_expressed or other.reason_expressed, + reason_text=self.reason_text or other.reason_text, + ) + + def to_dict(self) -> Dict[str, Any]: + return { + "driving": self.driving, + "walking": self.walking, + "transit": self.transit, + "time_pressured": self.time_pressured, + "reason_expressed": self.reason_expressed, + "reason_text": self.reason_text, + } + + @classmethod + def from_dict(cls, d: Optional[Dict[str, Any]]) -> "CallerSignal": + d = d or {} + return cls( + driving=bool(d.get("driving")), + walking=bool(d.get("walking")), + transit=bool(d.get("transit")), + time_pressured=bool(d.get("time_pressured")), + reason_expressed=bool(d.get("reason_expressed")), + reason_text=str(d.get("reason_text") or ""), + ) + + +def classify_utterance(text: str) -> CallerSignal: + """Extract a CallerSignal from one user turn. O(len(text)) regex.""" + if not text: + return CallerSignal() + sig = CallerSignal( + driving=bool(_MOBILITY_DRIVING.search(text)), + walking=bool(_MOBILITY_WALKING.search(text)), + transit=bool(_MOBILITY_TRANSIT.search(text)), + time_pressured=bool(_TIME_PRESSURE.search(text)), + ) + m = _REASON_CUE.search(text) + if m: + sig.reason_expressed = True + # Take the remainder of the sentence as the reason paraphrase + # — a cheap heuristic good enough for the phase machine. + start = m.end() + remainder = text[start:].strip() + end = len(remainder) + for stop in (".", "?", "!", "\n"): + idx = remainder.find(stop) + if idx != -1 and idx < end: + end = idx + sig.reason_text = remainder[:end].strip()[:140] + return sig diff --git a/backend/app/persona_call/directive.py b/backend/app/persona_call/directive.py new file mode 100644 index 0000000000000000000000000000000000000000..addf3d75589ce3f19f236150069c481b5ad09c31 --- /dev/null +++ b/backend/app/persona_call/directive.py @@ -0,0 +1,365 @@ +""" +Per-turn system-suffix composer. + +**Core invariant of this whole module** (the product promise): + + The persona's own system prompt is NEVER touched. `compose()` + only produces a short suffix that is appended as a SEPARATE + system message in addition to whatever the /v1/chat/completions + endpoint already injects for the persona. When the call ends, + the suffix is gone; the persona's chat behavior is identical to + what it was before the call started. + +The suffix is assembled from five small paragraphs. Each paragraph +is independently testable and individually gated by the current +phase / time / caller-context, so on an uneventful turn the suffix +is only ~4 lines long. + +The five stages, in assembly order: + + 1. _base_phone_frame persona-neutral phone register + 2. _phase_frame opening / topic / pre_closing / closed + 3. _context_frame late-night / driving / resumed-topic + 4. _repetition_frame ledger-enforced variety + 5. _latency_frame filler rule for thinking pauses +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from . import closing, repeat, state as state_mod, store +from .config import PersonaCallConfig +from .context import CallEnv, CallerSignal +from .facets import VoiceFacets + + +# ── public result type ─────────────────────────────────────────────── + +@dataclass +class ComposedDirective: + """Return value of :func:`compose`. Two independently useful fields: + + * ``system_suffix`` — the text we append as an additional system + message. If :class:`PersonaCallConfig`.apply is False (shadow + mode), this is logged but not actually sent to the model. + * ``post_directives`` — the list of individual rules we applied, + one string per rule, in the order composed. Dumped into + ``persona_call_directives.post_directives`` for observability. + """ + session_id: str + phase: str + turn_index: int + system_suffix: str + post_directives: List[str] = field(default_factory=list) + + +# ── paragraph builders ──────────────────────────────────────────────── + +def _base_phone_frame(facets: VoiceFacets) -> str: + """Persona-neutral phone register. Same for every persona; the + persona's identity is NOT asserted here — that stays in the + persona's own system prompt which is already active.""" + return ( + "[PHONE CALL MODE — appended only for the duration of this call]\n" + "You are on a live phone call. Spoken, not typed. " + "No bullet points, no stage directions (never output '*smiles*' " + "or similar), no markdown headers. " + f"Keep replies in the {facets.typical_reply_sentences}-sentence range, " + f"≤ {facets.sentence_max_words} words per sentence. " + f"Tone: {facets.tone_line}." + ) + + +def _phase_frame( + *, + phase: str, + turn_index: int, + skipped_how_are_you: bool, + asked_reason_fallback: bool, + facets: VoiceFacets, + env: CallEnv, + caller: CallerSignal, + pre_closing_trigger: Optional[str], + cfg: PersonaCallConfig, +) -> tuple[str, List[str]]: + """Return (paragraph, rule_names_applied) for the current phase.""" + rules: List[str] = [] + + if phase == state_mod.OPENING: + # Turn 0 — Schegloff summons-answer. + if turn_index == 0: + rules.append("opening.summons_answer") + return ( + "This is turn 0. Reply with ONE short line of recognition " + "— acknowledge you're there, use the caller's name if you " + "know it, ≤ 6 words. Do NOT launch into topic.", + rules, + ) + # Turn ≥ 1 — decide on "how are you" exactly once. + if not skipped_how_are_you and turn_index <= 2: + # Skip "how are you" if: late hour, caller is pressed, or + # caller already expressed a reason. + should_skip = ( + env.local_hour >= cfg.how_are_you_late_hour + or caller.time_pressured + or caller.reason_expressed + ) + if should_skip: + rules.append("opening.skip_how_are_you") + return ( + "Skip the 'how are you?' exchange this time — the " + "caller is either pressed, late, or already on-topic. " + "Move directly to hearing them out.", + rules, + ) + rules.append("opening.how_are_you_once") + return ( + "One brief round of 'how are you?' is appropriate now. " + "Ask once, let them answer, then pause for their reason " + "for calling. Do not stack this with another question.", + rules, + ) + # Opening past turn 2 with no reason and no fallback asked. + if ( + not caller.reason_expressed + and not asked_reason_fallback + and turn_index >= cfg.reason_fallback_turn + ): + rules.append("opening.reason_fallback_ask") + return ( + "The caller has not said why they're calling yet. " + "Ask gently, in one sentence: something like " + "'what's on your mind today?' or 'what's up?'", + rules, + ) + rules.append("opening.default") + return ( + "Still in the opening. Keep replies short and oriented to " + "the caller's lead.", + rules, + ) + + if phase == state_mod.TOPIC: + rules.append("topic.default") + return ( + "Mid-call. Be responsive: if the caller asked a question, " + "answer it before asking one back. Ask at most ONE follow-up " + "per turn — phone calls tolerate fewer questions per turn " + "than chat does.", + rules, + ) + + if phase == state_mod.PRE_CLOSING: + rules.append("closing.pre_closing_handshake") + return closing.pre_closing_directive(facets, pre_closing_trigger), rules + + if phase == state_mod.CLOSED: + rules.append("closing.terminal") + return closing.terminal_directive(facets), rules + + return "", rules + + +def _context_frame( + env: CallEnv, caller: CallerSignal, cfg: PersonaCallConfig, +) -> tuple[str, List[str]]: + """Situational rules. Paragraphs only emit when the predicate fires, + so uneventful turns get no context frame at all.""" + lines: List[str] = [] + rules: List[str] = [] + + # Late-night brevity (decision-fatigue bias). + late = ( + env.local_hour >= cfg.late_night_brevity_hour + or env.local_hour <= cfg.late_night_brevity_hour_end + ) + if late: + rules.append("context.late_night_brevity") + lines.append( + f"Local clock: {env.local_hour:02d}:00 for the caller. " + "It's late — keep replies to one sentence when you can, " + "offer to continue tomorrow for anything complex." + ) + + # Driving — safety/attention bias. + if caller.driving: + rules.append("context.driving") + lines.append( + "The caller is DRIVING. One-sentence answers, no multi-part " + "questions, never require a yes/no decision unless it's " + "genuinely urgent." + ) + elif caller.walking or caller.transit: + rules.append("context.mobile") + lines.append( + "The caller is mobile. Keep it brief and well-paced — they " + "may lose signal or attention momentarily." + ) + + # Weeks-since-last-call resumption (fire at most once per call; the + # persona_call_state doesn't need a separate flag because this only + # fires when the topic hasn't been acknowledged yet — the composer + # sees the same state each turn until the persona says something). + if env.weeks_since_last_call >= 3: + rules.append("context.resumption") + lines.append( + f"It has been ≈{env.weeks_since_last_call} weeks since this " + "caller last spoke with you. One natural acknowledgment of " + "the gap is appropriate — do NOT make it the whole turn, " + "and say it only once this call." + ) + if env.resumed_from_topic: + rules.append("context.resume_topic") + lines.append( + f"Unfinished topic from a prior call: {env.resumed_from_topic!r}. " + "Offer to pick it back up if natural, but don't force it." + ) + + return ("\n".join(lines) if lines else ""), rules + + +def _repetition_frame( + state: Dict[str, Any], facets: VoiceFacets, +) -> tuple[str, List[str]]: + text = repeat.forbid_clause(state, facets) + return (text, ["repeat.ledger"] if text else []) + + +def _latency_frame(facets: VoiceFacets) -> tuple[str, List[str]]: + thinking = ", ".join(facets.thinking_tokens[:3]) + return ( + "If you need a beat to think, start your reply with a short " + f"filler from: {thinking}. Never produce a long silence — " + "callers read silence on the phone as trouble.", + ["latency.filler_rule"], + ) + + +# ── public compose() ────────────────────────────────────────────────── + +def compose( + *, + session_id: str, + persona_id: Optional[str], + user_text: str, + env: CallEnv, + cfg: PersonaCallConfig, + facets: Optional[VoiceFacets] = None, +) -> ComposedDirective: + """Build the per-turn directive. + + 1. Advances the phase machine on the user utterance. + 2. Composes the five paragraphs. + 3. Persists the directive in shadow mode OR applies it. + """ + from .facets import for_persona_id + _facets = facets if facets is not None else for_persona_id(persona_id) + + # Step 1: phase machine. + new_state = state_mod.advance_on_user_utterance( + session_id=session_id, + user_text=user_text, + reason_fallback_turn=cfg.reason_fallback_turn, + ) + phase = str(new_state.get("phase") or state_mod.OPENING) + turn_index = int(new_state.get("turn_index") or 0) + + # The caller_context column is already updated by the phase machine, + # so re-derive the CallerSignal from it for the composer. + caller = CallerSignal.from_dict( + new_state.get("caller_context") + if isinstance(new_state.get("caller_context"), dict) else {} + ) + + # Step 2: five paragraphs. + rules_all: List[str] = [] + parts: List[str] = [_base_phone_frame(_facets)] + rules_all.append("base.phone_frame") + + phase_para, r = _phase_frame( + phase=phase, + turn_index=turn_index - 1, # phase machine incremented already + skipped_how_are_you=bool(new_state.get("skipped_how_are_you")), + asked_reason_fallback=bool(new_state.get("asked_reason_fallback")), + facets=_facets, + env=env, + caller=caller, + pre_closing_trigger=( + str(new_state.get("pre_closing_trigger") or "") + if new_state.get("pre_closing_trigger") else None + ), + cfg=cfg, + ) + if phase_para: + parts.append(phase_para) + rules_all.extend(r) + + ctx_para, r = _context_frame(env, caller, cfg) + if ctx_para: + parts.append(ctx_para) + rules_all.extend(r) + + rep_para, r = _repetition_frame(new_state, _facets) + if rep_para: + parts.append(rep_para) + rules_all.extend(r) + + lat_para, r = _latency_frame(_facets) + parts.append(lat_para) + rules_all.extend(r) + + system_suffix = "\n\n".join(parts).strip() + + # Step 3: persist + (maybe) apply. + store.record_directive( + session_id=session_id, + turn_index=turn_index, + phase=phase, + applied=cfg.apply, + system_suffix=system_suffix, + post_directives=rules_all, + ) + + # If we asked 'how are you' or the reason-fallback, set the flags + # so we don't do it again. + if "opening.how_are_you_once" in rules_all: + state_mod.mark_how_are_you_decision(session_id, skipped=False) + if "opening.skip_how_are_you" in rules_all: + state_mod.mark_how_are_you_decision(session_id, skipped=True) + if "opening.reason_fallback_ask" in rules_all: + state_mod.mark_reason_fallback_asked(session_id) + + return ComposedDirective( + session_id=session_id, + phase=phase, + turn_index=turn_index, + system_suffix=system_suffix if cfg.apply else "", + post_directives=rules_all, + ) + + +def record_persona_reply( + *, + session_id: str, + persona_id: Optional[str], + reply: str, + cfg: PersonaCallConfig, +) -> None: + """Called after the LLM reply comes back. Rolls the anti-repetition + ledger forward so the next turn can forbid the just-used token.""" + from .facets import for_persona_id + _facets = for_persona_id(persona_id) + state = store.ensure_state(session_id) + new_acks, new_openers = repeat.record_reply( + state, + reply, + _facets, + cfg_ack_window=cfg.ack_ledger_window, + cfg_opener_window=cfg.opener_ledger_window, + ) + store.update_state( + session_id, + recent_acks=new_acks, + recent_openers=new_openers, + ) diff --git a/backend/app/persona_call/facets.py b/backend/app/persona_call/facets.py new file mode 100644 index 0000000000000000000000000000000000000000..430cd2d5dafdb97b51c70d57301ec7365197cb13 --- /dev/null +++ b/backend/app/persona_call/facets.py @@ -0,0 +1,180 @@ +""" +Per-persona ``voice_facets`` — non-destructive voice personalization. + +Facets live in an OPTIONAL field on the persona manifest +(``persona.voice_facets``). Any existing persona with no facets gets +the neutral-English default — its chat behavior is completely +unchanged either way. + +The facets object drives four things at runtime: + + * backchannel token pools (``mm``, ``yeah``, ``right`` etc.) + * pre-closing + closing token pools + * filler / thinking tokens for the latency hook + * scalar voice knobs (warmth, pace, formality) interpolated into + the per-turn system suffix as raw numbers — the model reads the + number, not an adjective. This avoids the same three adjectives + bleeding into every reply. + +The *base system prompt* of the persona is never touched. This file +never rewrites a persona; it only reads the optional facets field. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + + +# ── Default facets — applied when a persona ships no voice_facets ─── +# +# Values chosen from the CA literature + common-sense phone norms. +# Token pools kept short on purpose: fewer options → easier anti- +# repetition tracking → cleaner entropy metric. + +_DEFAULT_ACKS = ["mm", "mm-hm", "yeah", "right", "got it", "okay"] +_DEFAULT_THINKING = ["hmm", "let me think", "one sec", "hold on"] +_DEFAULT_PRE_CLOSE = ["okay", "alright", "so", "well"] +_DEFAULT_CLOSING = ["take care", "talk soon", "bye for now", "later"] + + +@dataclass(frozen=True) +class VoiceFacets: + """Immutable snapshot for one persona. + + All scalar knobs are clamped on load so downstream code can trust + the range. Token pools are deduplicated + lowercased to keep the + anti-repetition ledger case-insensitive. + """ + + warmth: float # 0 clinical … 1 affectionate + pace: float # 0.25 … 2.0 speech-rate multiplier + formality: float # 0 casual … 1 formal + humor: float # 0 none … 1 playful + self_disclosure: float # 0 reserved … 1 volunteers a lot + sentence_max_words: int + typical_reply_sentences: List[int] + language_register: str + ack_tokens: List[str] + thinking_tokens: List[str] + pre_closing_tokens: List[str] + closing_tokens: List[str] + backchannel_rate: float # 0..2, multiplier on default cadence + + @property + def tone_line(self) -> str: + """One-line summary of scalar knobs, ready to drop into the + per-turn system suffix. Numeric so the model interpolates + rather than falling back on the same adjective.""" + return ( + f"warmth {self.warmth:.1f} · pace {self.pace:.2f} · " + f"formality {self.formality:.1f} · humor {self.humor:.1f}" + ) + + +def _clamp(x: float, lo: float, hi: float) -> float: + try: + return max(lo, min(hi, float(x))) + except (TypeError, ValueError): + return lo + + +def _tokens(raw: Any, default: List[str]) -> List[str]: + """Normalize a token-pool field: list[str], lowercase, de-dup, + keep stable order. Returns the default if input is empty / bad.""" + if not isinstance(raw, list) or not raw: + return list(default) + seen, out = set(), [] + for item in raw: + if not isinstance(item, str): + continue + t = item.strip().lower() + if t and t not in seen: + seen.add(t) + out.append(t) + return out if out else list(default) + + +def default_facets() -> VoiceFacets: + """Neutral English phone-call default. What an unmodified persona + gets — indistinguishable from today's behavior except for the phone + context itself.""" + return VoiceFacets( + warmth=0.5, + pace=1.0, + formality=0.4, + humor=0.3, + self_disclosure=0.3, + sentence_max_words=22, + typical_reply_sentences=[1, 2, 3], + language_register="en-US-casual", + ack_tokens=list(_DEFAULT_ACKS), + thinking_tokens=list(_DEFAULT_THINKING), + pre_closing_tokens=list(_DEFAULT_PRE_CLOSE), + closing_tokens=list(_DEFAULT_CLOSING), + backchannel_rate=1.0, + ) + + +def from_manifest(manifest: Optional[Dict[str, Any]]) -> VoiceFacets: + """Parse a persona manifest dict into a VoiceFacets. + + Any missing / malformed field falls back to the default. Safe to + call with ``None`` or ``{}`` — returns :func:`default_facets` in + those cases. Never raises; a broken facets block must not break a + call. + """ + d = default_facets() + if not isinstance(manifest, dict): + return d + raw = manifest.get("voice_facets") + if not isinstance(raw, dict): + return d + + sig = raw.get("signature_tokens") or {} + if not isinstance(sig, dict): + sig = {} + + reply_ns = raw.get("typical_reply_sentences") + if isinstance(reply_ns, list) and reply_ns: + reply_ns = [int(n) for n in reply_ns if isinstance(n, (int, float))] + else: + reply_ns = list(d.typical_reply_sentences) + + return VoiceFacets( + warmth=_clamp(raw.get("warmth", d.warmth), 0.0, 1.0), + pace=_clamp(raw.get("pace", d.pace), 0.25, 2.0), + formality=_clamp(raw.get("formality", d.formality), 0.0, 1.0), + humor=_clamp(raw.get("humor", d.humor), 0.0, 1.0), + self_disclosure=_clamp( + raw.get("self_disclosure", d.self_disclosure), 0.0, 1.0 + ), + sentence_max_words=int(raw.get("sentence_max_words", d.sentence_max_words)), + typical_reply_sentences=reply_ns, + language_register=str(raw.get("language_register", d.language_register)), + ack_tokens=_tokens(sig.get("acknowledgments"), d.ack_tokens), + thinking_tokens=_tokens(sig.get("thinking"), d.thinking_tokens), + pre_closing_tokens=_tokens(sig.get("pre_closing"), d.pre_closing_tokens), + closing_tokens=_tokens(sig.get("closings"), d.closing_tokens), + backchannel_rate=_clamp(raw.get("backchannel_rate", 1.0), 0.0, 2.0), + ) + + +# ── Runtime lookup — never mutates persona storage ─────────────────── + +def for_persona_id(persona_id: Optional[str]) -> VoiceFacets: + """Load facets for a persona without touching persona storage. + + Looks up the persona's manifest (if any) via the existing personas + module. Any error returns the neutral default — persona_call must + degrade silently when persona storage has not yet been initialized + (e.g. early tests). + """ + if not persona_id: + return default_facets() + try: + # Lazy import to avoid any import-time coupling to personas. + from ..personas.repo import get_persona # type: ignore + manifest = get_persona(persona_id) + except Exception: + return default_facets() + return from_manifest(manifest) diff --git a/backend/app/persona_call/latency.py b/backend/app/persona_call/latency.py new file mode 100644 index 0000000000000000000000000000000000000000..02e0d35fd68d528e7c235352533cc83a1a556a60 --- /dev/null +++ b/backend/app/persona_call/latency.py @@ -0,0 +1,107 @@ +""" +Pre-emptive filler scheduler — "hmm, one sec…" within the 700 ms +Stivers trouble threshold. + +Because the MVP chat path is *non-streaming* (the compat endpoint +returns 501 on stream=true), the only way to cover long LLM latency +is to emit a server-originated filler BEFORE the real reply arrives. + +Usage (from ws.py inside a turn):: + + async with FillerScheduler(ws=..., facets=facets, cfg=cfg, + session_id=sid) as sched: + reply = await turn.run_turn(...) + # If the turn took longer than cfg.filler_emit_after_ms, the + # scheduler already emitted one 'assistant.filler' event to the + # client. Either way, the real reply follows. + +Per the design doc: filler tokens are persona-specific with a +sensible universal fallback (covered by ``VoiceFacets.thinking_tokens``, +whose default pool is ["hmm", "let me think", "one sec", "hold on"]). +""" +from __future__ import annotations + +import asyncio +import random +import time +from typing import Any, Callable, Dict, List, Optional + +from .config import PersonaCallConfig +from .facets import VoiceFacets + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +class FillerScheduler: + """Async context-manager that emits at most ONE filler event per + turn, ``filler_emit_after_ms`` after entering the context. Cancel + on exit — if the LLM returned faster, no event is sent. + + The event is emitted via the ``send`` callable injected at + construction, not by calling ``ws.send_text`` directly, so this + module stays free of fastapi imports and is trivially unit-testable. + """ + + def __init__( + self, + *, + send: Callable[[Dict[str, Any]], "asyncio.Future"], + facets: VoiceFacets, + cfg: PersonaCallConfig, + session_id: str, + recent_thinking: Optional[List[str]] = None, + ) -> None: + self._send = send + self._facets = facets + self._cfg = cfg + self._sid = session_id + self._recent = list(recent_thinking or []) + self._task: Optional[asyncio.Task] = None + self.emitted: bool = False + self.emitted_token: Optional[str] = None + + def _pick_token(self) -> Optional[str]: + pool = [t for t in self._facets.thinking_tokens if t] + if not pool: + return None + last = (self._recent[-1].lower() if self._recent else "") + candidates = [t for t in pool if t.lower() != last] or pool + return random.choice(candidates) + + async def _fire(self) -> None: + delay_ms = max(0, int(self._cfg.filler_emit_after_ms)) + try: + await asyncio.sleep(delay_ms / 1000.0) + except asyncio.CancelledError: + return + token = self._pick_token() + if not token: + return + try: + await self._send({ + "type": "assistant.filler", + "payload": { + "token": token, + "ts": _now_ms(), + "session_id": self._sid, + }, + }) + self.emitted = True + self.emitted_token = token + except Exception: + # Best-effort — never let a filler failure break the turn. + pass + + async def __aenter__(self) -> "FillerScheduler": + self._task = asyncio.create_task(self._fire()) + return self + + async def __aexit__(self, exc_type, exc, tb) -> None: + if self._task is not None: + self._task.cancel() + try: + await self._task + except Exception: + pass diff --git a/backend/app/persona_call/openings.py b/backend/app/persona_call/openings.py new file mode 100644 index 0000000000000000000000000000000000000000..7160d0449803a654b0f8186b72fa55362186dcd2 --- /dev/null +++ b/backend/app/persona_call/openings.py @@ -0,0 +1,334 @@ +""" +persona_call.openings — "theory of answering" template bank. + +Adds the deterministic first-turn answer the persona uses when a call +connects. Backed by conversation-analysis research on phone openings +(Schegloff 1968, Hopper 1992) and cross-cultural data (Liddicoat +2021, ch. 9); augmented with call-center industry practice for the +formal-register templates. + +Why a template bank instead of "let the LLM improvise" +------------------------------------------------------ +Phone openings are the single most ritualized slot in human +conversation — every culture has a narrow, low-entropy answer +inventory. A free-form LLM greeting is both slower (latency hurts +most on turn 1) and less natural (LLMs over-generate: "Hello there! +What a wonderful day to connect with you…"). A curated bank of +short templates fires in <5 ms, is persona-aware, is rotated so the +same caller never hears the same greeting twice in a row, and is +trivially auditable. + +Design constraints +------------------ +1. **Additive.** This module is new; nothing else changes. The + existing ``directive.compose()`` uses it opportunistically in + the opening phase only. +2. **Persona-preserving.** A template is a *seed*, not a mandate: + the chosen text flows through the persona's own TTS voice and + the LLM never sees it as an instruction to copy. If a persona + ships ``voice_facets.signature_tokens.openings``, that override + wins; templates from this module are used only when the persona + declines to ship its own. +3. **Deterministic under tests.** Selection is seeded by + (session_id, turn_index) so the same call always picks the same + template — regression tests don't flake. +4. **Rotated.** A separate persisted ledger + (``persona_call_state.recent_openings``) already tracks the + last few opener tokens the persona used; we extend it to cover + full template IDs so cross-call repetition is avoided too. + +Template families (drawn from Hopper 1992 + call-center practice) +----------------------------------------------------------------- +- **summons_answer** — the minimalist acknowledge-you've-been-summoned + move: "Hello?", "Yes?", "Mm?" (Schegloff 1968, turn 1). +- **identification** — name-forward answer, the default for most + non-intimate contexts: "Atlas speaking.", "This is Atlas." +- **greeting_plus_name** — warmer, the default personal-call opener + in English: "Hi, Atlas here.", "Hey — Atlas." +- **time_of_day** — morning/evening variants: "Good evening, Atlas.", + "Morning — Atlas.". +- **intimate_reopen** — for repeat callers (weeks_since_last_call + < 2): "Hey you.", "Oh hi!", "Good to hear from you.". +- **call_center_formal** — for personas with formality ≥ 0.75: + "Thank you for calling. This is Atlas, how can I help?" and + variants. The *only* family that includes a how-may-I-help tail. +- **late_night_brief** — after 22:00 local: "Hey…", "Mm, yeah?", + "Yeah — hi.". Short on purpose. + +Each template carries a ``precond`` callable that decides whether it +fits the current ``(facets, env, state)``. Selection is a two-pass +filter+weight: first drop ineligible templates, then weight the +survivors by persona fit. +""" + +from __future__ import annotations + +import hashlib +import random +from dataclasses import dataclass, field +from typing import Callable, Iterable, List, Optional + +from .context import CallEnv +from .facets import VoiceFacets + + +# ── Template record ────────────────────────────────────────────────── + +@dataclass(frozen=True) +class OpeningTemplate: + """One row in the template bank. + + Attributes + ---------- + id + Stable identifier persisted in the anti-repetition ledger. + Short snake_case. + family + One of the seven families in the module docstring. Used by + selection weighting + tests. + text + The template string. ``{name}`` is interpolated with the + persona's display name (or swallowed if empty). No other + interpolation — keeping templates 100% literal means a + reviewer can read the bank and know exactly what can ship. + formality_range + Inclusive ``(lo, hi)`` on the persona's formality facet. + Templates outside the persona's range are filtered out. + precond + Final predicate run against ``(facets, env)``. Returns True + iff the template is eligible *right now*. Typical uses: + late-night variants only after 22:00, intimate variants only + for recurring callers. + """ + id: str + family: str + text: str + formality_range: tuple = (0.0, 1.0) + precond: Callable[[VoiceFacets, CallEnv], bool] = field( + default=lambda _f, _e: True, + ) + + +# ── Precondition helpers — readable and reusable. ──────────────────── + +def _any_time(_f: VoiceFacets, _e: CallEnv) -> bool: + return True + +def _recurring(_f: VoiceFacets, env: CallEnv) -> bool: + """Fire only when this caller has spoken to this persona recently. + Guards against a cold-call receiving the intimate family.""" + return 0 <= env.weeks_since_last_call < 2 + +def _late_night(_f: VoiceFacets, env: CallEnv) -> bool: + return env.local_hour >= 22 or env.local_hour < 6 + +def _morning(_f: VoiceFacets, env: CallEnv) -> bool: + return 5 <= env.local_hour < 11 + +def _afternoon(_f: VoiceFacets, env: CallEnv) -> bool: + return 11 <= env.local_hour < 17 + +def _evening(_f: VoiceFacets, env: CallEnv) -> bool: + return 17 <= env.local_hour < 22 + + +# ── The bank ───────────────────────────────────────────────────────── +# Deliberately small — 23 templates across 7 families. Adding more is +# cheap but every addition increases the risk of one slipping through +# review in a voice that doesn't fit the persona. Keep the bar high. + +OPENING_BANK: tuple = ( + # 1. summons_answer — the Schegloff minimum. Any formality. + OpeningTemplate("sa_hello", "summons_answer", "Hello?"), + OpeningTemplate("sa_hello_open", "summons_answer", "Hello — who's this?"), + OpeningTemplate("sa_yes", "summons_answer", "Yes?", + formality_range=(0.0, 0.6)), + OpeningTemplate("sa_yeah", "summons_answer", "Yeah?", + formality_range=(0.0, 0.45)), + + # 2. identification — name-forward. Default for 0.3 ≤ formality ≤ 0.8. + OpeningTemplate("id_name_speaking", "identification", + "{name} speaking.", + formality_range=(0.4, 1.0)), + OpeningTemplate("id_this_is", "identification", + "This is {name}.", + formality_range=(0.35, 1.0)), + OpeningTemplate("id_hello_name", "identification", + "Hello, this is {name}.", + formality_range=(0.4, 1.0)), + + # 3. greeting_plus_name — warmer, casual. Default personal-call. + OpeningTemplate("gn_hi_here", "greeting_plus_name", + "Hi, {name} here.", + formality_range=(0.2, 0.75)), + OpeningTemplate("gn_hey_name", "greeting_plus_name", + "Hey — {name}.", + formality_range=(0.0, 0.6)), + OpeningTemplate("gn_hi_hi", "greeting_plus_name", + "Hi hi.", + formality_range=(0.0, 0.55)), + + # 4. time_of_day — used only at matching hour, amortised across + # the bank so it never dominates. + OpeningTemplate("td_morning", "time_of_day", + "Morning — {name}.", + precond=_morning), + OpeningTemplate("td_good_morning", "time_of_day", + "Good morning, {name} speaking.", + formality_range=(0.5, 1.0), precond=_morning), + OpeningTemplate("td_afternoon", "time_of_day", + "Good afternoon — {name}.", + formality_range=(0.4, 1.0), precond=_afternoon), + OpeningTemplate("td_evening", "time_of_day", + "Good evening.", + formality_range=(0.4, 1.0), precond=_evening), + + # 5. intimate_reopen — recurring callers only. + OpeningTemplate("int_hey_you", "intimate_reopen", + "Hey you.", + formality_range=(0.0, 0.5), precond=_recurring), + OpeningTemplate("int_oh_hi", "intimate_reopen", + "Oh, hi!", + formality_range=(0.0, 0.6), precond=_recurring), + OpeningTemplate("int_good_to_hear", "intimate_reopen", + "Hey — good to hear from you.", + formality_range=(0.15, 0.7), precond=_recurring), + + # 6. call_center_formal — high-formality only; the only family + # that volunteers help. + OpeningTemplate("cc_thank_you_calling", "call_center_formal", + "Thank you for calling. This is {name}, how can I help?", + formality_range=(0.75, 1.0)), + OpeningTemplate("cc_name_how_help", "call_center_formal", + "{name} speaking — how can I help?", + formality_range=(0.7, 1.0)), + OpeningTemplate("cc_standing_by", "call_center_formal", + "Hello, {name} here — what can I do for you?", + formality_range=(0.65, 1.0)), + + # 7. late_night_brief — just picked up the phone, mid-yawn. + OpeningTemplate("ln_hey", "late_night_brief", + "Hey…", + formality_range=(0.0, 0.55), precond=_late_night), + OpeningTemplate("ln_yeah_hi", "late_night_brief", + "Yeah — hi.", + formality_range=(0.0, 0.5), precond=_late_night), + OpeningTemplate("ln_mm_yeah", "late_night_brief", + "Mm, yeah?", + formality_range=(0.0, 0.45), precond=_late_night), +) + + +# ── Public API ─────────────────────────────────────────────────────── + +def _interpolate(tpl: str, name: str) -> str: + """Render ``{name}`` in the template. If the caller has no usable + display name, swallow the placeholder cleanly so we never ship + literal ``{name}``.""" + safe = (name or "").strip() + if not safe: + # Drop the placeholder + surrounding whitespace around it so + # "This is {name}." → "This is." wouldn't be ugly. We replace + # with a near-natural collapse. + return ( + tpl.replace(", this is {name}.", ".") + .replace(" this is {name}.", ".") + .replace(" — {name}.", ".") + .replace("{name} speaking.", "Speaking.") + .replace("{name} here.", "Here.") + .replace("{name}", "") + .strip() + ) + return tpl.replace("{name}", safe) + + +def eligible_templates( + facets: VoiceFacets, + env: CallEnv, + *, + bank: Iterable[OpeningTemplate] = OPENING_BANK, + forbidden_ids: Iterable[str] = (), +) -> List[OpeningTemplate]: + """Filter the bank to what fits the persona + environment + ledger. + + Two-pass filter: + 1. Formality range contains ``facets.formality``. + 2. ``precond(facets, env)`` returns True. + 3. ``id`` not in ``forbidden_ids`` (the anti-repetition ledger). + """ + banned = set(forbidden_ids or ()) + out: List[OpeningTemplate] = [] + for t in bank: + lo, hi = t.formality_range + if not (lo <= facets.formality <= hi): + continue + if not t.precond(facets, env): + continue + if t.id in banned: + continue + out.append(t) + return out + + +def _seed(session_id: str, turn_index: int) -> int: + """Stable seed so tests don't flake. + + Uses blake2b rather than hash() because CPython randomizes + ``hash()`` between processes (PYTHONHASHSEED), which makes the + selected template drift under test isolation and is a subtle + source of flakes. + """ + h = hashlib.blake2b( + f"{session_id}:{turn_index}".encode("utf-8"), digest_size=8, + ) + return int.from_bytes(h.digest(), "big") + + +def choose( + facets: VoiceFacets, + env: CallEnv, + *, + session_id: str, + turn_index: int = 1, + forbidden_ids: Iterable[str] = (), + bank: Iterable[OpeningTemplate] = OPENING_BANK, +) -> Optional[OpeningTemplate]: + """Pick one eligible template. + + Returns the persona's own signature opener when ``facets`` ships a + custom pool — otherwise picks from the filtered bank. Never + returns an ``OpeningTemplate`` that matches a forbidden id. + + If no eligible template survives the filter (rare — would require + an unusual formality + time-of-day combination and a ledger full + of every normally-eligible id), this returns ``None`` and the + caller should fall back to ``"Hello?"``, which is why the + summons_answer family has no preconditions. + """ + pool = eligible_templates(facets, env, bank=bank, forbidden_ids=forbidden_ids) + if not pool: + return None + rnd = random.Random(_seed(session_id, turn_index)) + return rnd.choice(pool) + + +def render( + facets: VoiceFacets, + env: CallEnv, + *, + persona_display_name: str = "", + session_id: str, + turn_index: int = 1, + forbidden_ids: Iterable[str] = (), +) -> Optional[str]: + """End-to-end helper: pick + interpolate. Returns None on empty + pool so the caller can fall back safely.""" + tpl = choose( + facets, env, + session_id=session_id, + turn_index=turn_index, + forbidden_ids=forbidden_ids, + ) + if tpl is None: + return None + return _interpolate(tpl.text, persona_display_name) diff --git a/backend/app/persona_call/repeat.py b/backend/app/persona_call/repeat.py new file mode 100644 index 0000000000000000000000000000000000000000..e1f3b9977104d2a43504dec9e52f67fd05561200 --- /dev/null +++ b/backend/app/persona_call/repeat.py @@ -0,0 +1,128 @@ +""" +Anti-repetition ledger. + +Three windowed deques per session, all backed by the JSON columns on +``persona_call_state``: + + recent_acks last N acknowledgment tokens the persona used + recent_openers last N first-words of the persona's turns + recent_closings last N pre-closing tokens + +Read API returns the set to FORBID in the next turn's directive; write +API is called after the reply comes back and extracts the new token via +a regex-first / LLM-never extractor so it stays cheap. + +No LLM calls anywhere here — the extraction is regex + token-pool +membership. If the persona's reply starts with a token that isn't in +its own voice_facets pool, we store the first lowercased word anyway; +it still anchors the entropy metric. +""" +from __future__ import annotations + +import re +from typing import Dict, List, Optional, Set, Tuple + +from .facets import VoiceFacets + + +_WORD_BOUNDARY = re.compile(r"^[\s\W]*([a-z][a-z\-']*)", re.I) + + +def _first_word(reply: str) -> Optional[str]: + if not reply: + return None + m = _WORD_BOUNDARY.match(reply.strip()) + if not m: + return None + return m.group(1).lower() + + +def _roll(deque: List[str], token: str, size: int) -> List[str]: + out = list(deque or []) + if token: + out.append(token) + if len(out) > size: + out = out[-size:] + return out + + +# ── READ: directive paragraph that forbids recent tokens ───────────── + +def forbid_clause( + state: Dict[str, object], + facets: VoiceFacets, + *, + max_len: int = 80, +) -> str: + """Return the one-line directive paragraph about recent tokens. + + Empty string when there's nothing to forbid yet — keeps the + per-turn suffix short on early turns.""" + acks = state.get("recent_acks") or [] + openers = state.get("recent_openers") or [] + if not acks and not openers: + return "" + + dropped: Set[str] = set() + for t in acks: + if isinstance(t, str) and t: + dropped.add(t.lower()) + for t in openers: + if isinstance(t, str) and t: + dropped.add(t.lower()) + + if not dropped: + return "" + + pool = [t for t in facets.ack_tokens if t.lower() not in dropped] + fresh_hint = "" + if pool: + fresh_hint = f" Prefer a fresh one from: {', '.join(pool[:4])}." + + forbidden = sorted(dropped)[: max_len // 5] # rough clamp + return ( + "Avoid repeating your recent openers / acknowledgments: " + f"{', '.join(forbidden)}." + fresh_hint + ) + + +# ── WRITE: record the persona's reply into the ledger ──────────────── + +def record_reply( + state: Dict[str, object], + reply: str, + facets: VoiceFacets, + cfg_ack_window: int, + cfg_opener_window: int, +) -> Tuple[List[str], List[str]]: + """Roll the ledger forward. Returns the new (recent_acks, + recent_openers) lists for ``store.update_state``.""" + token = _first_word(reply) or "" + acks = state.get("recent_acks") or [] + openers = state.get("recent_openers") or [] + + # An ack is anything that appears in the persona's ack pool AND + # appears in the first three words. That matches how human phone + # acks actually work ("mm yeah I hear you"): the ack is the front + # of the turn, not a whole separate utterance. + front = " ".join((reply or "").lower().strip().split()[:3]) + matched_ack = None + for pool_token in facets.ack_tokens: + if pool_token in front: + matched_ack = pool_token + break + + new_acks = _roll(list(acks), matched_ack or "", cfg_ack_window) + new_openers = _roll(list(openers), token, cfg_opener_window) + return new_acks, new_openers + + +def record_closing( + state: Dict[str, object], + token: str, + window: int = 3, +) -> List[str]: + """Roll the closings deque. Called when the closing state machine + chooses a pre-closing token.""" + prev = state.get("recent_closings") or [] + return _roll(list(prev), token.lower() if token else "", window) diff --git a/backend/app/persona_call/router.py b/backend/app/persona_call/router.py new file mode 100644 index 0000000000000000000000000000000000000000..f4f207a378963a984a6169d42927cbe787a8b660 --- /dev/null +++ b/backend/app/persona_call/router.py @@ -0,0 +1,224 @@ +""" +Public HTTP surface for persona_call. + +Five endpoints, all minor — this module is primarily called from +``voice_call/ws.py`` at runtime. These REST routes are for +observability + admin tuning + client facets lookup. + +Mount point in main.py is ONE feature-flagged try/except block. +With ``PERSONA_CALL_ENABLED=false`` :func:`build_router` returns +an empty router — every path 404s, zero schema is created. +""" +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from fastapi import ( + APIRouter, Cookie, Depends, Header, HTTPException, Query, +) +from pydantic import BaseModel + +from . import facets as facets_mod +from . import store +from .config import PersonaCallConfig + + +# ── auth passthrough (matches main._scoped_user_or_none) ───────────── + +def _resolve_user( + authorization: str, + homepilot_session: Optional[str], +) -> dict: + from ..users import ( + ensure_users_tables, _validate_token, + get_or_create_default_user, count_users, + ) + ensure_users_tables() + token = "" + if authorization and authorization.lower().startswith("bearer "): + token = authorization.split(" ", 1)[1].strip() + if not token and homepilot_session: + token = homepilot_session.strip() + user = _validate_token(token) if token else None + if user: + return user + if count_users() > 1: + raise HTTPException(status_code=401, detail="Authentication required") + return get_or_create_default_user() + + +# ── response models ────────────────────────────────────────────────── + +class FacetsResp(BaseModel): + persona_id: Optional[str] + warmth: float + pace: float + formality: float + humor: float + self_disclosure: float + sentence_max_words: int + typical_reply_sentences: List[int] + language_register: str + signature_tokens: Dict[str, List[str]] + backchannel_rate: float + + +class StateResp(BaseModel): + session_id: str + phase: str + turn_index: int + skipped_how_are_you: bool + asked_reason_fallback: bool + reason_for_call: str + pre_closing_trigger: Optional[str] + recent_acks: List[str] + recent_openers: List[str] + recent_closings: List[str] + caller_context: Dict[str, Any] + + +class DirectiveResp(BaseModel): + session_id: str + turn_index: int + phase: str + applied: bool + system_suffix: str + post_directives: List[str] + + +# ── router builder ─────────────────────────────────────────────────── + +def build_router(cfg: PersonaCallConfig) -> APIRouter: + """Construct the FastAPI router. Empty if ``cfg.enabled`` is + False so main.py's mount is always a safe no-op.""" + r = APIRouter(prefix="/v1/persona-call", tags=["persona-call"]) + if not cfg.enabled: + return r + + @r.get("/facets/{persona_id}", response_model=FacetsResp) + def get_facets(persona_id: str): + f = facets_mod.for_persona_id(persona_id) + return FacetsResp( + persona_id=persona_id, + warmth=f.warmth, + pace=f.pace, + formality=f.formality, + humor=f.humor, + self_disclosure=f.self_disclosure, + sentence_max_words=f.sentence_max_words, + typical_reply_sentences=f.typical_reply_sentences, + language_register=f.language_register, + signature_tokens={ + "acknowledgments": list(f.ack_tokens), + "thinking": list(f.thinking_tokens), + "pre_closing": list(f.pre_closing_tokens), + "closings": list(f.closing_tokens), + }, + backchannel_rate=f.backchannel_rate, + ) + + @r.get("/state/{session_id}", response_model=StateResp) + def get_state( + session_id: str, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + """Runtime state for the session. Owner-bound via the + underlying voice_call_sessions row — if the session doesn't + belong to the caller, return 404 (probe-safe).""" + user = _resolve_user(authorization, homepilot_session) + # Ownership is enforced by looking up the voice_call_sessions + # row first. If that is missing or foreign, we 404 before + # touching persona_call_state at all. + try: + from ..voice_call.store import get_session_for_owner + vc_row = get_session_for_owner(session_id, user["id"]) + except Exception: + vc_row = None + if vc_row is None: + raise HTTPException(status_code=404, detail="Session not found") + s = store.get_state(session_id) + if s is None: + # Session exists but no persona_call state yet — return a + # zero-ed row rather than 404 so clients can poll safely + # before the first turn. + return StateResp( + session_id=session_id, + phase="opening", + turn_index=0, + skipped_how_are_you=False, + asked_reason_fallback=False, + reason_for_call="", + pre_closing_trigger=None, + recent_acks=[], + recent_openers=[], + recent_closings=[], + caller_context={}, + ) + return StateResp( + session_id=s["session_id"], + phase=str(s.get("phase") or "opening"), + turn_index=int(s.get("turn_index") or 0), + skipped_how_are_you=bool(s.get("skipped_how_are_you")), + asked_reason_fallback=bool(s.get("asked_reason_fallback")), + reason_for_call=str(s.get("reason_for_call") or ""), + pre_closing_trigger=s.get("pre_closing_trigger"), + recent_acks=list(s.get("recent_acks") or []), + recent_openers=list(s.get("recent_openers") or []), + recent_closings=list(s.get("recent_closings") or []), + caller_context=dict(s.get("caller_context") or {}), + ) + + @r.get("/last-directive/{session_id}", response_model=DirectiveResp) + def get_last_directive( + session_id: str, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + user = _resolve_user(authorization, homepilot_session) + try: + from ..voice_call.store import get_session_for_owner + vc_row = get_session_for_owner(session_id, user["id"]) + except Exception: + vc_row = None + if vc_row is None: + raise HTTPException(status_code=404, detail="Session not found") + d = store.last_directive(session_id) + if d is None: + raise HTTPException(status_code=404, detail="No directive yet") + return DirectiveResp( + session_id=d["session_id"], + turn_index=int(d["turn_index"]), + phase=str(d["phase"]), + applied=bool(d["applied"]), + system_suffix=str(d["system_suffix"] or ""), + post_directives=list(d["post_directives"] or []), + ) + + @r.post("/reset/{session_id}") + def reset_session( + session_id: str, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + """Wipe the anti-repetition ledgers for a session. Owner-bound. + Used by QA + tests; harmless in production because the phase + machine just starts fresh from 'opening'.""" + user = _resolve_user(authorization, homepilot_session) + try: + from ..voice_call.store import get_session_for_owner + vc_row = get_session_for_owner(session_id, user["id"]) + except Exception: + vc_row = None + if vc_row is None: + raise HTTPException(status_code=404, detail="Session not found") + store.update_state( + session_id, + recent_acks=[], + recent_openers=[], + recent_closings=[], + ) + return {"ok": True, "session_id": session_id} + + return r diff --git a/backend/app/persona_call/state.py b/backend/app/persona_call/state.py new file mode 100644 index 0000000000000000000000000000000000000000..426c3d7df6260ab57e5ce198b9820d1ac99c9e26 --- /dev/null +++ b/backend/app/persona_call/state.py @@ -0,0 +1,99 @@ +""" +Phase state machine. + +Five phases, deterministic transitions: + + opening ─(reason expressed OR 'how are you' cycle done)─▶ topic + topic ─(closing trigger)─────────────────────────────▶ pre_closing + pre_closing ─(user terminal)──────────────────────────────▶ closed + pre_closing ─(user recants / new topic)──────────────────▶ topic + closed (terminal) + +Phase is the single source of truth for which paragraph +:mod:`directive` injects into the per-turn system suffix. +""" +from __future__ import annotations + +from typing import Dict, Optional + +from . import closing, context, store + + +OPENING = "opening" +TOPIC = "topic" +PRE_CLOSING = "pre_closing" +CLOSED = "closed" + + +def advance_on_user_utterance( + *, + session_id: str, + user_text: str, + reason_fallback_turn: int, +) -> Dict[str, object]: + """Called by the WS orchestrator BEFORE composing the directive. + + Updates the phase, turn_index, caller_context, and reason flags + based on the just-received user utterance. Returns the refreshed + state row so the composer has the latest view. + """ + state = store.ensure_state(session_id) + turn_index = int(state.get("turn_index") or 0) + phase = state.get("phase") or OPENING + + # Caller signal extraction — accumulate into caller_context. + prev_signal = context.CallerSignal.from_dict( + state.get("caller_context") if isinstance( + state.get("caller_context"), dict) else {} + ) + fresh = context.classify_utterance(user_text) + merged = prev_signal.merge(fresh) + + # Phase transition rules. + pre_closing_trigger = state.get("pre_closing_trigger") + + if phase in (OPENING, TOPIC) and closing.is_pre_closing_trigger(user_text): + phase = PRE_CLOSING + pre_closing_trigger = user_text.strip() + elif phase == PRE_CLOSING and closing.is_terminal(user_text): + phase = CLOSED + elif phase == OPENING: + # Move to 'topic' if the user stated a reason, OR if we've + # already done one "how are you" cycle (2 user turns in + # opening without a reason). + if merged.reason_expressed: + phase = TOPIC + elif turn_index >= 2: + # After two turns in 'opening' without a stated reason, + # the phase-machine forces 'topic' — the persona will + # ask the reason-fallback question via the composer. + phase = TOPIC + + # Persist. + updated = store.update_state( + session_id, + phase=phase, + turn_index=turn_index + 1, + caller_context=merged.to_dict(), + pre_closing_trigger=pre_closing_trigger, + ) + return updated + + +def mark_how_are_you_decision(session_id: str, *, skipped: bool) -> None: + """Record that the composer made a 'how are you' decision for this + session — idempotent; the deciding composer path may run multiple + turns but should only fire once.""" + store.update_state(session_id, skipped_how_are_you=skipped) + + +def mark_reason_fallback_asked(session_id: str) -> None: + """Record that the composer already asked the reason-fallback + question, so it never asks twice.""" + store.update_state(session_id, asked_reason_fallback=True) + + +def get(session_id: str) -> Optional[Dict[str, object]]: + """Thin pass-through to the store — avoids callers importing + store directly.""" + return store.get_state(session_id) diff --git a/backend/app/persona_call/store.py b/backend/app/persona_call/store.py new file mode 100644 index 0000000000000000000000000000000000000000..b363e8fe02d7fe3e627f49ad32e5db197454aee6 --- /dev/null +++ b/backend/app/persona_call/store.py @@ -0,0 +1,241 @@ +""" +SQLite persistence for persona_call. + +Two tables, both additive and empty when the feature flag is off: + + persona_call_state per-session runtime state (phase, turn_index, + ledgers, caller_context). + + persona_call_directives shadow-mode audit: every composed directive + is appended here so P1 (compose but don't + apply) produces a reviewable trail. + +Follows the same pattern as backend/app/voice_call/store.py and the +rest of HomePilot: direct sqlite3, per-call connections, lazy +``ensure_schema`` with a process-scoped guard so it's cheap after +first use. +""" +from __future__ import annotations + +import json +import sqlite3 +import time +from typing import Any, Dict, List, Optional + + +def _db_path() -> str: + from ..storage import _get_db_path + return _get_db_path() + + +def _conn() -> sqlite3.Connection: + con = sqlite3.connect(_db_path()) + con.row_factory = sqlite3.Row + return con + + +_SCHEMA_READY = False + + +def ensure_schema() -> None: + """Create persona_call tables if missing. Safe to call any number + of times. No-op on subsequent calls within the same process.""" + global _SCHEMA_READY + if _SCHEMA_READY: + return + con = _conn() + cur = con.cursor() + cur.executescript( + """ + CREATE TABLE IF NOT EXISTS persona_call_state ( + session_id TEXT PRIMARY KEY, + phase TEXT NOT NULL DEFAULT 'opening', + turn_index INTEGER NOT NULL DEFAULT 0, + skipped_how_are_you INTEGER NOT NULL DEFAULT 0, + asked_reason_fallback INTEGER NOT NULL DEFAULT 0, + reason_for_call TEXT NOT NULL DEFAULT '', + last_backchannel_ms INTEGER NOT NULL DEFAULT 0, + recent_acks TEXT NOT NULL DEFAULT '[]', + recent_openers TEXT NOT NULL DEFAULT '[]', + recent_closings TEXT NOT NULL DEFAULT '[]', + caller_context TEXT NOT NULL DEFAULT '{}', + pre_closing_trigger TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS persona_call_directives ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + turn_index INTEGER NOT NULL, + phase TEXT NOT NULL, + applied INTEGER NOT NULL DEFAULT 1, + system_suffix TEXT NOT NULL, + post_directives TEXT NOT NULL DEFAULT '[]', + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS ix_pcd_session + ON persona_call_directives (session_id, turn_index); + """ + ) + con.commit() + con.close() + _SCHEMA_READY = True + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +# ── persona_call_state CRUD ─────────────────────────────────────────── + +def get_state(session_id: str) -> Optional[Dict[str, Any]]: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + "SELECT * FROM persona_call_state WHERE session_id = ?", + (session_id,), + ) + row = cur.fetchone() + con.close() + if row is None: + return None + d = {k: row[k] for k in row.keys()} + # Deserialize JSON columns. + for col in ("recent_acks", "recent_openers", "recent_closings", + "caller_context"): + try: + d[col] = json.loads(d.get(col) or "[]") + except Exception: + d[col] = [] if col != "caller_context" else {} + return d + + +def ensure_state(session_id: str) -> Dict[str, Any]: + """Upsert-shaped: returns the existing row, or creates a fresh one + in the 'opening' phase if this is the first turn.""" + existing = get_state(session_id) + if existing is not None: + return existing + now = _now_ms() + con = _conn() + cur = con.cursor() + cur.execute( + """ + INSERT OR IGNORE INTO persona_call_state + (session_id, created_at, updated_at) + VALUES (?, ?, ?) + """, + (session_id, now, now), + ) + con.commit() + con.close() + return get_state(session_id) # type: ignore[return-value] + + +def update_state(session_id: str, **changes: Any) -> Dict[str, Any]: + """Apply a partial update. JSON-typed columns accept Python lists / + dicts; everything else is a scalar.""" + if not changes: + return ensure_state(session_id) + + json_cols = {"recent_acks", "recent_openers", "recent_closings", + "caller_context"} + sets, params = [], [] + for col, val in changes.items(): + if col in json_cols: + val = json.dumps(val, separators=(",", ":")) + if col in ("skipped_how_are_you", "asked_reason_fallback"): + val = 1 if val else 0 + sets.append(f"{col} = ?") + params.append(val) + sets.append("updated_at = ?") + params.append(_now_ms()) + params.append(session_id) + + ensure_state(session_id) + con = _conn() + cur = con.cursor() + cur.execute( + f"UPDATE persona_call_state SET {', '.join(sets)} WHERE session_id = ?", + params, + ) + con.commit() + con.close() + return get_state(session_id) # type: ignore[return-value] + + +# ── persona_call_directives (shadow-mode audit) ─────────────────────── + +def record_directive( + *, + session_id: str, + turn_index: int, + phase: str, + applied: bool, + system_suffix: str, + post_directives: List[str], +) -> None: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + """ + INSERT INTO persona_call_directives + (session_id, turn_index, phase, applied, + system_suffix, post_directives, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + session_id, + int(turn_index), + phase, + 1 if applied else 0, + system_suffix, + json.dumps(post_directives, separators=(",", ":")), + _now_ms(), + ), + ) + con.commit() + con.close() + + +def last_directive(session_id: str) -> Optional[Dict[str, Any]]: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + """ + SELECT session_id, turn_index, phase, applied, + system_suffix, post_directives, created_at + FROM persona_call_directives + WHERE session_id = ? + ORDER BY id DESC + LIMIT 1 + """, + (session_id,), + ) + row = cur.fetchone() + con.close() + if row is None: + return None + d = {k: row[k] for k in row.keys()} + try: + d["post_directives"] = json.loads(d.get("post_directives") or "[]") + except Exception: + d["post_directives"] = [] + d["applied"] = bool(d.get("applied")) + return d + + +# ── test helper ─────────────────────────────────────────────────────── + +def _reset_for_tests() -> None: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute("DELETE FROM persona_call_directives") + cur.execute("DELETE FROM persona_call_state") + con.commit() + con.close() diff --git a/backend/app/studio/render_jobs.py b/backend/app/studio/render_jobs.py index 3ad0aa4e148a8cd25bb98c0bb3e7e56c6b03c0ec..e06e91ab5e1c5c0b674ef78ff3ed0269c300018d 100644 --- a/backend/app/studio/render_jobs.py +++ b/backend/app/studio/render_jobs.py @@ -53,6 +53,11 @@ class RenderJob: duration_sec: float = 0.0 width: int = 0 height: int = 0 + # Render-time audio / subtitle options, captured so /jobs/{id} + # responses can describe exactly how the file was produced. + audio_rate: float = 1.0 + audio_pitch: float = 1.0 + subtitles: str = "none" def to_dict(self) -> Dict[str, Any]: d = asdict(self) @@ -115,6 +120,9 @@ def submit_render( kind: RenderKind, preset: PlatformPreset, scenes: List[SceneInput], + audio_rate: float = 1.0, + audio_pitch: float = 1.0, + subtitles: str = "none", ) -> RenderJob: """Queue a render and return its job record. Worker thread starts immediately.""" job_id = _new_job_id() @@ -128,6 +136,9 @@ def submit_render( preset=preset, status="queued", output_path=output_path, + audio_rate=audio_rate, + audio_pitch=audio_pitch, + subtitles=subtitles, ) _store(job) @@ -169,6 +180,9 @@ def _run_job(job_id: str, scenes: List[SceneInput]) -> None: kind=job.kind, output_path=job.output_path or "", on_progress=_on_progress, + audio_rate=job.audio_rate, + audio_pitch=job.audio_pitch, + subtitles=job.subtitles, ) _update( job_id, diff --git a/backend/app/studio/render_mp4.py b/backend/app/studio/render_mp4.py index 5912be2721a6e2bfe9bcc5e3ba9ec27cb6bbf7e2..415b2a9ae009d9e5c5e188dabd48be163720bfe9 100644 --- a/backend/app/studio/render_mp4.py +++ b/backend/app/studio/render_mp4.py @@ -54,12 +54,17 @@ class SceneInput: Either ``image_path`` or ``video_path`` should be provided. If both are set, ``video_path`` wins (it is the higher-fidelity asset). If neither is set, the scene is rendered as a black frame of ``duration_sec``. + + ``narration`` is the source text used to synthesize subtitles when + the caller passes ``subtitles="burn_in"`` to :func:`render_scenes`. + It does not affect rendering when subtitles are off. """ idx: int duration_sec: float = 5.0 image_path: Optional[str] = None video_path: Optional[str] = None audio_path: Optional[str] = None + narration: str = "" @dataclass @@ -243,6 +248,119 @@ def _per_scene_args( return inputs, filters, f"v{scene.idx}" +def _audio_transform_chain(rate: float, pitch: float) -> Optional[str]: + """Build the ffmpeg audio-filter chain for the chosen rate + pitch. + + Returns None when both are 1.0 (caller skips the transform stage). + + Rate is applied with ``atempo``. Pitch uses the industry-standard + ``asetrate=48000*pitch, atempo=1/pitch`` compound so the tempo is + preserved while the pitch shifts. When both are non-unit we stack the + pitch compound first, then an outer atempo for rate. + + ``atempo`` only accepts 0.5–2.0 per stage; we clamp inputs upstream + to stay within that range so we never need to chain multiple stages. + """ + rate = max(0.5, min(2.0, float(rate or 1.0))) + pitch = max(0.5, min(2.0, float(pitch or 1.0))) + if rate == 1.0 and pitch == 1.0: + return None + parts: list[str] = [] + if pitch != 1.0: + parts.append(f"asetrate=48000*{pitch:.6f}") + parts.append(f"atempo={1.0 / pitch:.6f}") + if rate != 1.0: + parts.append(f"atempo={rate:.6f}") + return ",".join(parts) + + +def _ass_escape(text: str) -> str: + """Escape a string for embedding inside a libass SRT/ASS cue.""" + return (text or "").replace("\\", "\\\\").replace("{", "\\{").replace("}", "\\}") + + +def _srt_timestamp(seconds: float) -> str: + """Convert seconds to an SRT timestamp (HH:MM:SS,mmm).""" + if seconds < 0: + seconds = 0.0 + ms = int(round(seconds * 1000)) + hours, ms = divmod(ms, 3_600_000) + minutes, ms = divmod(ms, 60_000) + secs, ms = divmod(ms, 1000) + return f"{hours:02d}:{minutes:02d}:{secs:02d},{ms:03d}" + + +def _split_narration_for_subtitles(text: str, max_chars: int = 42) -> list[str]: + """Chunk narration into subtitle-sized lines. + + Subtitles read best at ~42 chars/line (YouTube's auto-caption default). + We split on sentence boundaries first, then on commas when a chunk is + still too long, falling back to hard-wrap at word boundaries. + """ + import re + if not text: + return [] + # Sentence split on ., !, ?, or newline. + sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+|\n+", text) if s.strip()] + chunks: list[str] = [] + for s in sentences: + if len(s) <= max_chars: + chunks.append(s) + continue + # Try splitting on commas first. + sub = [p.strip() for p in s.split(",") if p.strip()] + if all(len(p) <= max_chars for p in sub): + chunks.extend(sub) + continue + # Fallback: greedy word-wrap. + cur = "" + for word in s.split(): + if not cur: + cur = word + elif len(cur) + 1 + len(word) <= max_chars: + cur = f"{cur} {word}" + else: + chunks.append(cur) + cur = word + if cur: + chunks.append(cur) + return chunks + + +def _write_srt_for_scene( + narration: str, + duration_sec: float, + srt_path: Path, + *, + start_offset: float = 0.0, +) -> bool: + """Write an SRT file whose cues span [0, duration_sec] for a single scene. + + Returns True on success, False when the narration is empty (no SRT + should be emitted — and the caller should not try to burn an empty + subtitle file, which would fail). + """ + chunks = _split_narration_for_subtitles(narration) + if not chunks: + return False + dur = max(0.5, float(duration_sec or 5.0)) + # Distribute the duration evenly across chunks. Cap per-cue at 5s so + # slow voices don't hold a single cue on screen forever. + per = min(dur / len(chunks), 5.0) + lines: list[str] = [] + for i, chunk in enumerate(chunks): + start = start_offset + i * per + end = min(start_offset + dur, start + per) + if end <= start: + end = start + 0.1 + lines.append(str(i + 1)) + lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}") + lines.append(_ass_escape(chunk)) + lines.append("") + srt_path.write_text("\n".join(lines), encoding="utf-8") + return True + + def _has_audio(path: str) -> bool: """Probe a media file and return True iff it has at least one audio stream.""" try: @@ -277,6 +395,9 @@ def render_scenes( kind: RenderKind, output_path: str, on_progress: Optional[Callable[[float], None]] = None, + audio_rate: float = 1.0, + audio_pitch: float = 1.0, + subtitles: str = "none", ) -> RenderResult: """Render a list of scenes into a single MP4 at ``output_path``. @@ -284,6 +405,18 @@ def render_scenes( then concatenates the intermediates with ffmpeg's concat demuxer. This is slower than a single filter graph but far more robust against scenes with very different codecs / resolutions / sample rates / pixel formats. + + Extra knobs: + + - ``audio_rate`` (0.5–2.0, default 1.0): tempo multiplier for the + narration/audio track. Applied as ffmpeg ``atempo``. + - ``audio_pitch`` (0.5–2.0, default 1.0): pitch-shift multiplier. + Implemented via ``asetrate=48000*pitch, atempo=1/pitch`` so the + output duration is unchanged (tempo is preserved, pitch shifts). + - ``subtitles`` (``"burn_in"`` | ``"none"``, default ``"none"``): + when ``burn_in``, each scene's ``narration`` is chunked into an + SRT file and burned into the video via libass. A sidecar .srt + is also written next to the MP4 so editors can re-style. """ if not ffmpeg_available(): raise RuntimeError( @@ -294,6 +427,8 @@ def render_scenes( width, height, fps = _resolution_for_preset(preset) encode_args = _encode_args_for_kind(kind, preset) + audio_transform = _audio_transform_chain(audio_rate, audio_pitch) + burn_subtitles = (subtitles or "none").lower() == "burn_in" out = Path(output_path) out.parent.mkdir(parents=True, exist_ok=True) @@ -325,24 +460,61 @@ def render_scenes( image_path=image_local, video_path=video_local, audio_path=audio_local, + narration=scene.narration, ) - inputs, filters, _vlabel = _per_scene_args( + inputs, filters, vlabel = _per_scene_args( normalized, width, height, fps, scratch ) + # If subtitles are burning in, write a per-scene SRT into + # scratch and append a ``subtitles`` filter to the video chain. + # ffmpeg's subtitles filter uses libass under the hood. A + # sidecar .srt is also written next to the final MP4 below. + if burn_subtitles and scene.narration: + srt_path = scratch / f"scene_{i:04d}.srt" + wrote = _write_srt_for_scene( + scene.narration, + normalized.duration_sec, + srt_path, + ) + if wrote: + # Escape the path for ffmpeg's filter-arg parser. + esc = str(srt_path).replace("\\", "/").replace(":", r"\:") + force_style = ( + "Fontname=DejaVu Sans,Fontsize=28," + "PrimaryColour=&H00FFFFFF&," + "OutlineColour=&H80000000&," + "BorderStyle=3,Outline=2,Shadow=0,MarginV=48" + ) + # Replace the last video-chain step's output label so + # the subtitle filter sits at the end of the pipeline. + new_last = ( + f"[{vlabel}]subtitles='{esc}':" + f"force_style='{force_style}'[{vlabel}_sub]" + ) + filters.append(new_last) + vlabel = f"{vlabel}_sub" + # Audio handling: prefer explicit audio_path; otherwise reuse the # video's audio track if present; otherwise generate silence so # every clip has the same stream layout for the concat demuxer. audio_inputs: list[str] = [] audio_filter: list[str] = [] audio_label = "a0" + # Common tail applied to every audio path: resample to 48 kHz + # stereo + apply the optional rate/pitch transform the caller + # requested. Consolidating the tail here keeps the three + # branches below in sync. + tail = "aresample=48000,aformat=channel_layouts=stereo" + if audio_transform: + tail = f"{tail},{audio_transform}" if audio_local: audio_inputs += ["-i", audio_local] # Index 1 is the audio file (after the single video input). - audio_filter.append(f"[1:a]aresample=48000,aformat=channel_layouts=stereo[{audio_label}]") + audio_filter.append(f"[1:a]{tail}[{audio_label}]") elif video_local and _has_audio(video_local): - audio_filter.append(f"[0:a]aresample=48000,aformat=channel_layouts=stereo[{audio_label}]") + audio_filter.append(f"[0:a]{tail}[{audio_label}]") else: audio_inputs += [ "-f", "lavfi", @@ -362,7 +534,7 @@ def render_scenes( *inputs, *audio_inputs, "-filter_complex", filter_complex, - "-map", "[v0]", + "-map", f"[{vlabel}]", "-map", f"[{audio_label}]", "-r", str(fps), "-shortest", @@ -396,6 +568,36 @@ def render_scenes( ] _run(concat_cmd) + # Sidecar SRT: full-video timeline, cue timings chained across + # scenes. Written whenever burn_subtitles=True, so editors get the + # same timing they saw burned in. Named .srt. + if burn_subtitles: + sidecar = out.with_suffix(".srt") + running_offset = 0.0 + cue_idx = 1 + lines: list[str] = [] + for s in scenes: + dur = max(0.5, float(s.duration_sec or 5.0)) + chunks = _split_narration_for_subtitles(s.narration or "") + if chunks: + per = min(dur / len(chunks), 5.0) + for j, chunk in enumerate(chunks): + start = running_offset + j * per + end = min(running_offset + dur, start + per) + if end <= start: + end = start + 0.1 + lines.append(str(cue_idx)) + lines.append(f"{_srt_timestamp(start)} --> {_srt_timestamp(end)}") + lines.append(_ass_escape(chunk)) + lines.append("") + cue_idx += 1 + running_offset += dur + try: + sidecar.write_text("\n".join(lines), encoding="utf-8") + except OSError: + # Non-fatal — the burned-in subs are still present. + pass + if on_progress: on_progress(100.0) diff --git a/backend/app/studio/routes.py b/backend/app/studio/routes.py index 234af05a3bd6b367e81d435f78782254c63f15ec..d675b946bd3777232bf0e16cf2fa5b1734d8fb0a 100644 --- a/backend/app/studio/routes.py +++ b/backend/app/studio/routes.py @@ -2360,8 +2360,24 @@ class VideoMp4ExportRequest(BaseModel): ``kind`` selects the encode profile; the platform preset (canvas aspect, fps) is read from the StudioVideo record so the same project always renders to the same canvas regardless of the chosen profile. + + Audio / subtitle options are additive: the defaults reproduce the + original concat-only behavior so the endpoint stays backward + compatible for clients that don't send them. """ kind: Literal["mp4_plain", "mp4_youtube"] = "mp4_plain" + # Audio rate (tempo) multiplier applied during render, 0.5–2.0. 1.0 is + # untouched; <1 slows narration, >1 speeds it up. Duration of the + # output clip stretches / shrinks accordingly. + audio_rate: float = 1.0 + # Audio pitch shift, 0.5–2.0. 1.0 is untouched. Implemented with the + # industry-standard asetrate + atempo compound so tempo is preserved + # while only pitch shifts. + audio_pitch: float = 1.0 + # When "burn_in", scene narration is wrapped into timed SRT cues and + # burned into the video via ffmpeg's libass subtitle filter. A sidecar + # .srt is always written alongside the MP4 when subtitles != "none". + subtitles: Literal["burn_in", "none"] = "none" @router.post("/videos/{video_id}/export/mp4", status_code=202) @@ -2394,6 +2410,8 @@ def video_export_mp4( raise HTTPException(status_code=400, detail="Video has no scenes to render") # Translate StudioScene records into the renderer's SceneInput shape. + # ``narration`` is the source text for subtitles; it is the same field + # the Piper WASM wizard used to synthesize the uploaded audioUrl. inputs = [ SceneInput( idx=i, @@ -2401,20 +2419,100 @@ def video_export_mp4( image_path=s.imageUrl, video_path=s.videoUrl, audio_path=s.audioUrl, + narration=s.narration or "", ) for i, s in enumerate(scenes) ] + # Clamp audio-transform inputs. The renderer itself has additional + # defensive clamps, but rejecting absurd values at the API boundary + # makes error messages clearer. + audio_rate = max(0.5, min(2.0, float(req.audio_rate or 1.0))) + audio_pitch = max(0.5, min(2.0, float(req.audio_pitch or 1.0))) + job = render_jobs.submit_render( user_id=user["id"], video_id=video_id, kind=req.kind, preset=video.platformPreset, scenes=inputs, + audio_rate=audio_rate, + audio_pitch=audio_pitch, + subtitles=req.subtitles, ) return job.to_dict() +# ---------------------------------------------------------------------------- +# Narration upload +# ---------------------------------------------------------------------------- +# The Creator Studio export wizard synthesizes per-scene narration with the +# Piper WASM engine (in-browser), then uploads each WAV here. The server +# persists the file under UPLOAD_DIR/studio/narration/.wav and +# writes scene.audioUrl. During render the concat pipeline mixes this audio +# into the final MP4. + +from fastapi import UploadFile, File + + +@router.post("/videos/{video_id}/scenes/{scene_id}/narration") +async def upload_scene_narration( + video_id: str, + scene_id: str, + file: UploadFile = File(..., description="Audio file (Piper-synthesized WAV)"), + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), +): + """Accept a narration audio upload for a scene. Writes ``scene.audioUrl``.""" + _user = _resolve_user_or_401(authorization, homepilot_session) + + video = get_video(video_id) + if not video: + raise HTTPException(status_code=404, detail="Video not found") + scene = get_scene(scene_id) + if not scene or scene.videoId != video_id: + raise HTTPException(status_code=404, detail="Scene not found") + + # Piper produces 16-bit PCM WAV. Accept common audio types; the + # renderer normalizes anyway. + allowed_prefixes = ("audio/", "application/octet-stream") + ctype = (file.content_type or "").lower() + if ctype and not any(ctype.startswith(p) for p in allowed_prefixes): + raise HTTPException( + status_code=415, + detail=f"Unsupported narration content-type: {file.content_type}", + ) + + # Cap the upload at 50 MB — a minute of 48 kHz 16-bit stereo WAV is + # ~11 MB, so this is generous even for long scenes. + MAX_BYTES = 50 * 1024 * 1024 + payload = await file.read(MAX_BYTES + 1) + if len(payload) > MAX_BYTES: + raise HTTPException(status_code=413, detail="Narration file too large (>50 MB)") + if not payload: + raise HTTPException(status_code=400, detail="Empty narration file") + + from ..config import UPLOAD_DIR + narration_dir = os.path.join(UPLOAD_DIR, "studio", "narration") + os.makedirs(narration_dir, exist_ok=True) + # Use the scene_id as the filename so re-uploads replace the previous + # take cleanly and there is no growing pile of orphan wav files. + dest_path = os.path.join(narration_dir, f"{scene_id}.wav") + with open(dest_path, "wb") as fh: + fh.write(payload) + + # Write back via the existing scene update path so everything + # downstream (frontend list, exporter) reads a consistent record. + update_scene(scene_id, StudioSceneUpdate(audioUrl=dest_path)) + + return { + "ok": True, + "scene_id": scene_id, + "audioUrl": dest_path, + "bytes": len(payload), + } + + @router.get("/videos/{video_id}/export/jobs/{job_id}") def video_export_job_status( video_id: str, diff --git a/backend/app/voice_call/__init__.py b/backend/app/voice_call/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7652490fc91d347582396cf8dc9d3909750b04ea --- /dev/null +++ b/backend/app/voice_call/__init__.py @@ -0,0 +1,16 @@ +""" +HomePilot voice-call — additive backend module. + +Scope (MVP M1) +-------------- +Purely additive feature. With ``VOICE_CALL_ENABLED=false`` (the default) +this package is never imported into the running app and the DB tables it +needs are never created — dropping the module directory is sufficient to +remove the feature entirely. + +Public entry point is :func:`voice_call.router.build_router`; everything +else is internal. No existing backend module imports from this package. +""" +from __future__ import annotations + +__all__ = ["config", "router"] diff --git a/backend/app/voice_call/barge_in.py b/backend/app/voice_call/barge_in.py new file mode 100644 index 0000000000000000000000000000000000000000..7281fe34c8478e286e765df0965d7b2619b2d468 --- /dev/null +++ b/backend/app/voice_call/barge_in.py @@ -0,0 +1,123 @@ +""" +Per-session barge-in token registry. + +One process-local ``dict`` keyed by ``session_id``; each entry carries +a single :class:`BargeInToken` representing the currently-active +assistant turn. The token exposes an ``asyncio.Event`` the streaming +turn runner polls between chunks — when the client sends +``user.barge_in`` the WS handler calls :func:`cancel_active` which +flips the event and the generator exits cleanly on its next poll. + +The module is deliberately small and state-machine-only: + + * no HTTP, no DB, no asyncio transport — unit-testable in + milliseconds, + * ``cancel_active`` refuses cancels for stale ``turn_id`` values so + a barge-in racing a new turn is a silent no-op, + * ``cancel`` is idempotent (``asyncio.Event.set()`` is idempotent), + * ``clear`` wipes a session on WS close so a resume starts fresh. + +Contract is documented in +``docs/analysis/voice-call-streaming-design.md`` § 4.3. +""" +from __future__ import annotations + +import asyncio +import threading +from dataclasses import dataclass, field +from typing import Dict, Optional + + +@dataclass +class BargeInToken: + """Per-turn cancellation handle. + + Created by :func:`new_token` when the WS handler starts streaming + an assistant turn; polled by the streaming runner; flipped by + :func:`cancel_active` when the user barges in. + """ + + turn_id: str + _ev: asyncio.Event = field(default_factory=asyncio.Event) + + def cancel(self) -> None: + """Flip the event. Idempotent.""" + self._ev.set() + + def is_cancelled(self) -> bool: + """Cheap read the streaming runner calls between chunks.""" + return self._ev.is_set() + + async def wait(self) -> None: + """Optional helper for callers that want to ``asyncio.wait`` + on cancel rather than poll.""" + await self._ev.wait() + + +# ── process-local registry ──────────────────────────────────────────── +# One active token per session. A single lock because the hot path +# (``cancel_active``) is O(1) — no contention worth sharding for. + +_lock = threading.Lock() +_active: Dict[str, BargeInToken] = {} + + +def new_token(session_id: str, turn_id: str) -> BargeInToken: + """Register ``turn_id`` as the active turn for ``session_id``. + + Any previous active token for this session is replaced — if a + turn started without the previous one clearing (which would be + a bug upstream) we fail gracefully rather than leak. + """ + token = BargeInToken(turn_id=turn_id) + with _lock: + _active[session_id] = token + return token + + +def cancel_active(session_id: str, turn_id: str) -> bool: + """Cancel the active turn for ``session_id`` IFF its ``turn_id`` + matches the argument. + + Returns True on a successful cancel, False if the active turn has + a different id (stale barge-in) or no turn is active. + """ + with _lock: + token = _active.get(session_id) + if token is None or token.turn_id != turn_id: + return False + # Release the lock before setting — the event set is itself + # thread-safe, and keeping the lock narrow keeps the hot path + # fast under contention. + token.cancel() + return True + + +def get_active(session_id: str) -> Optional[BargeInToken]: + """Read the currently-active token without mutating. + + Used by the WS handler to find the active ``turn_id`` when the + client sends a ``transcript.partial`` (secondary barge-in signal) + without an explicit ``turn_id``. + """ + with _lock: + return _active.get(session_id) + + +def clear_session(session_id: str) -> None: + """Drop all tokens for ``session_id`` — called on WS close. + + If a turn was mid-stream we cancel it first so the streaming + runner exits promptly; the handler is responsible for emitting + the final envelope. + """ + with _lock: + token = _active.pop(session_id, None) + if token is not None and not token.is_cancelled(): + token.cancel() + + +def _reset_for_tests() -> None: + """Test hook — wipe the whole registry between cases.""" + with _lock: + _active.clear() diff --git a/backend/app/voice_call/config.py b/backend/app/voice_call/config.py new file mode 100644 index 0000000000000000000000000000000000000000..e969535dfd54cf4c3c0d60a09e93207050285c9d --- /dev/null +++ b/backend/app/voice_call/config.py @@ -0,0 +1,92 @@ +""" +voice_call runtime configuration. + +Every knob is an environment variable with a safe default. The feature +ships **off by default** — set ``VOICE_CALL_ENABLED=true`` to opt in. + +Privacy defaults are deliberately conservative per the review note +("treat storage/privacy as first-class from day 1"): +- ``VOICE_CALL_STORE_TRANSCRIPTS`` defaults to ``false``. +- Metadata persistence (sessions + events) is enabled by default, but + events only record event *types*, not full audio or raw user text. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass + + +def _bool_env(name: str, default: bool) -> bool: + raw = os.getenv(name) + if raw is None: + return default + return raw.strip().lower() in ("1", "true", "yes", "on") + + +def _int_env(name: str, default: int) -> int: + raw = os.getenv(name) + if raw is None: + return default + try: + return int(raw) + except (TypeError, ValueError): + return default + + +@dataclass(frozen=True) +class VoiceCallConfig: + """Resolved runtime knobs. Immutable; re-import to refresh in tests.""" + + # Master flag. When false, ``router.build_router()`` returns an empty + # router and ``store.migrate()`` is a no-op — the feature is entirely + # latent. + enabled: bool + + # WebSocket transport for the turn bridge. Off => HTTP-only polling + # fallback (not yet implemented, so if disabled the feature is + # effectively off even if ``enabled`` is true). + websocket_enabled: bool + + # Provider strategy. Only "internal" is wired at MVP; the string is + # reserved for future external-realtime adapters. + provider: str + + # Policy caps (server-enforced — never trust the client). + max_duration_sec: int + idle_timeout_sec: int + resume_window_sec: int + max_concurrent_per_user: int + session_create_rate_per_min: int + + # Privacy. Turn transcripts are NOT persisted by default. + store_transcripts: bool + artifact_retention_days: int + + # Phase 2 + 3 streaming (token-level LLM streaming + barge-in). + # Defaults off — when false, every byte on the wire and every + # code path taken is identical to today. Independent of + # ``enabled`` / ``websocket_enabled`` so the streaming path + # can be dark-shipped alongside unary turns for measurement. + streaming_enabled: bool + # Barge-in is an optional sub-feature of streaming; can be + # disabled independently if the VAD tap is misbehaving on a + # given deployment. + barge_in_enabled: bool + + +def load() -> VoiceCallConfig: + """Resolve config from env vars every call. Cheap; keep fresh for tests.""" + return VoiceCallConfig( + enabled=_bool_env("VOICE_CALL_ENABLED", False), + websocket_enabled=_bool_env("VOICE_CALL_WEBSOCKET_ENABLED", True), + provider=os.getenv("VOICE_CALL_PROVIDER", "internal").strip() or "internal", + max_duration_sec=_int_env("VOICE_CALL_MAX_DURATION_SEC", 45 * 60), + idle_timeout_sec=_int_env("VOICE_CALL_IDLE_TIMEOUT_SEC", 30), + resume_window_sec=_int_env("VOICE_CALL_RESUME_WINDOW_SEC", 20), + max_concurrent_per_user=_int_env("VOICE_CALL_MAX_CONCURRENT", 2), + session_create_rate_per_min=_int_env("VOICE_CALL_CREATE_RATE", 10), + store_transcripts=_bool_env("VOICE_CALL_STORE_TRANSCRIPTS", False), + artifact_retention_days=_int_env("VOICE_CALL_ARTIFACT_TTL_DAYS", 30), + streaming_enabled=_bool_env("VOICE_CALL_STREAMING_ENABLED", False), + barge_in_enabled=_bool_env("VOICE_CALL_BARGE_IN_ENABLED", True), + ) diff --git a/backend/app/voice_call/models.py b/backend/app/voice_call/models.py new file mode 100644 index 0000000000000000000000000000000000000000..2c4d4b82036e72fbc010b3c3b710d6f6f69a5d80 --- /dev/null +++ b/backend/app/voice_call/models.py @@ -0,0 +1,112 @@ +""" +voice_call request / response / event schemas. + +Kept small on purpose: the public surface is half a dozen shapes. If a +new field is needed, add it optional so the client stays forward- +compatible with older backends (same rule the rest of HomePilot follows). +""" +from __future__ import annotations + +from typing import Any, Dict, List, Literal, Optional +from pydantic import BaseModel, Field + + +EntryMode = Literal["header_button", "long_press", "hint_tap", "resume"] + +SessionStatus = Literal[ + "connecting", + "live", + "interrupted", # WS dropped, within resume window + "ending", + "ended", +] + + +class DeviceInfo(BaseModel): + platform: Optional[str] = None # "web" / "ios" / "android" / "desktop" + app_version: Optional[str] = None + + +class CreateSessionReq(BaseModel): + conversation_id: Optional[str] = None + persona_id: Optional[str] = None + entry_mode: EntryMode = "header_button" + device_info: DeviceInfo = Field(default_factory=DeviceInfo) + + +class Capabilities(BaseModel): + """What the server promises this session can do. Keep truthful — any + false promise here turns into a UX bug users feel immediately.""" + # At MVP turn-based only. Full duplex (true barge-in) reads as false + # until we add server-side audio, per the review note. + interruptions: bool = False + barge_in: bool = False + # Assistant replies come as ``transcript.final`` only — no partials. + # This matches the current ``/v1/chat/completions`` behavior which + # rejects streaming with a 501 (openai_compat_endpoint.py:477). + transcript_live: bool = False + # Phase 2/3 opt-in — set True on the response when + # ``VOICE_CALL_STREAMING_ENABLED`` is on. Unaware clients ignore + # the field and stay on the unary path. + streaming: bool = False + + +class CreateSessionResp(BaseModel): + session_id: str + ws_url: str + resume_token: str # short-lived; only accepted by ws handshake + expires_at: int # epoch ms; WS must open before this + max_duration_sec: int + capabilities: Capabilities + + +class Session(BaseModel): + """Public session record. Mirrors the DB row without resume_token or + raw payloads, so an accidental log of this shape never leaks + credentials.""" + id: str + user_id: str + conversation_id: Optional[str] = None + persona_id: Optional[str] = None + entry_mode: EntryMode + status: SessionStatus + started_at: int + ended_at: Optional[int] = None + ended_reason: Optional[str] = None + duration_sec: Optional[int] = None + + +class EndSessionResp(BaseModel): + session_id: str + status: SessionStatus + ended_at: int + ended_reason: str + duration_sec: int + + +class HintsResp(BaseModel): + """Discoverability copy + timings. Server is the source of truth so + the product team can tune thresholds without a frontend ship.""" + hint_enabled: bool = True + show_on_mic_tap: bool = True + auto_hide_ms: int = 1400 + hold_threshold_ms: int = 220 + # Which entry modes this build accepts. Lets us A/B or disable a + # mode without a client change. + accepted_entry_modes: List[EntryMode] = Field( + default_factory=lambda: ["header_button", "long_press", "hint_tap"] + ) + + +# ── WebSocket event envelope ────────────────────────────────────────── + +class WsEvent(BaseModel): + """Every message over the WS — both directions — wraps into this. + + ``seq`` is monotonic per session for events the server originates; + client-originated events don't need it (server ignores it if set). + """ + type: str + seq: Optional[int] = None + ts: Optional[int] = None + payload: Dict[str, Any] = Field(default_factory=dict) diff --git a/backend/app/voice_call/policy.py b/backend/app/voice_call/policy.py new file mode 100644 index 0000000000000000000000000000000000000000..cb32dd7850acc4a97eccecbafc184d79e1b225e4 --- /dev/null +++ b/backend/app/voice_call/policy.py @@ -0,0 +1,73 @@ +""" +Server-enforced policy for voice-call sessions. + +Never trust the client; every cap here is enforced before a session row +is inserted or a WebSocket is accepted. Counters live in the DB via +``store`` so they survive process restarts and apply across workers. +""" +from __future__ import annotations + +from . import store +from .config import VoiceCallConfig + + +class PolicyError(Exception): + """Raised when a caller violates a policy cap. Carries an HTTP status + and a short machine-readable code so the router can return a clean + JSON error without leaking internals.""" + + def __init__(self, http_status: int, code: str, detail: str) -> None: + super().__init__(detail) + self.http_status = http_status + self.code = code + self.detail = detail + + +def check_session_create(*, user_id: str, cfg: VoiceCallConfig) -> None: + """Called before :func:`store.create_session`. Raises ``PolicyError`` + on any violation. + + Checks applied, in order of cheapness: + 1. ``VOICE_CALL_ENABLED`` — feature gate. + 2. Concurrent session cap (default 2). Stops the "infinite parallel + calls" abuse vector. + 3. Rate limit on session creates (default 10 / 60 s). Stops script + kiddies from hammering ``/sessions``. + """ + if not cfg.enabled: + raise PolicyError( + http_status=404, code="voice_call_disabled", + detail="Voice call feature is not enabled on this server.", + ) + + active = store.count_active_sessions_for_user(user_id) + if active >= cfg.max_concurrent_per_user: + raise PolicyError( + http_status=409, code="too_many_active_sessions", + detail=( + f"You already have {active} active voice call(s). " + f"End one before starting a new one." + ), + ) + + recent = store.recent_session_creates_for_user( + user_id, window_sec=60, + ) + if recent >= cfg.session_create_rate_per_min: + raise PolicyError( + http_status=429, code="rate_limited", + detail=( + f"Too many voice call starts in the last minute " + f"(limit {cfg.session_create_rate_per_min})." + ), + ) + + +def session_hard_cap_exceeded(session_row: dict, cfg: VoiceCallConfig) -> bool: + """True if the session has exceeded its hard duration cap. + The WS orchestrator checks this every heartbeat to force ``ending``.""" + started = int(session_row.get("started_at") or 0) + if started <= 0: + return False + now_ms = int(__import__("time").time() * 1000) + return (now_ms - started) >= cfg.max_duration_sec * 1000 diff --git a/backend/app/voice_call/router.py b/backend/app/voice_call/router.py new file mode 100644 index 0000000000000000000000000000000000000000..366a774271e841d6618544b64f409b764bc0ec6c --- /dev/null +++ b/backend/app/voice_call/router.py @@ -0,0 +1,223 @@ +""" +Public HTTP + WebSocket surface. + +All endpoints live under ``/v1/voice-call/*``. The router is built +dynamically so that with ``VOICE_CALL_ENABLED=false`` it returns an +empty router — no endpoints, no 500s, no accidental exposure. + +Mount from main.py ONCE, always behind the feature flag:: + + from .voice_call.router import build_router, build_ws_router + from .voice_call.config import load as _vc_load + _vc_cfg = _vc_load() + if _vc_cfg.enabled: + app.include_router(build_router(_vc_cfg)) + app.include_router(build_ws_router(_vc_cfg)) +""" +from __future__ import annotations + +import logging +import os +from typing import Optional + +from fastapi import ( + APIRouter, Cookie, Depends, Header, HTTPException, + Query, WebSocket, +) +from fastapi.responses import JSONResponse + +from . import service, store, ws as ws_mod +from .config import VoiceCallConfig +from .models import ( + Capabilities, CreateSessionReq, CreateSessionResp, + EndSessionResp, HintsResp, Session, +) +from .policy import PolicyError + + +logger = logging.getLogger("voice_call.http") + + +# ── shared auth resolver (matches main._scoped_user_or_none) ────────── +# Kept inline to avoid a circular import. Same semantics: single-user +# installs auto-resolve to the default user; multi-user installs require +# a bearer token. + +def _resolve_user( + authorization: str, + homepilot_session: Optional[str], +) -> dict: + from ..users import ( + ensure_users_tables, _validate_token, + get_or_create_default_user, count_users, + ) + ensure_users_tables() + token = "" + if authorization and authorization.lower().startswith("bearer "): + token = authorization.split(" ", 1)[1].strip() + if not token and homepilot_session: + token = homepilot_session.strip() + user = _validate_token(token) if token else None + if user: + return user + if count_users() > 1: + raise HTTPException(status_code=401, detail="Authentication required") + return get_or_create_default_user() + + +# ── helpers ──────────────────────────────────────────────────────────── + +def _public_session(row: dict) -> Session: + """Strip the resume_token out before the dict leaves the server.""" + duration = None + if row.get("ended_at") and row.get("started_at"): + duration = int((int(row["ended_at"]) - int(row["started_at"])) / 1000) + return Session( + id=row["id"], + user_id=row["user_id"], + conversation_id=row.get("conversation_id"), + persona_id=row.get("persona_id"), + entry_mode=row["entry_mode"], + status=row["status"], + started_at=int(row["started_at"]), + ended_at=int(row["ended_at"]) if row.get("ended_at") else None, + ended_reason=row.get("ended_reason"), + duration_sec=duration, + ) + + +def _ws_url_for(session_id: str) -> str: + """The WS URL the client should open. Override via env for + containerized deployments where 127.0.0.1 is wrong.""" + base = ( + os.getenv("VOICE_CALL_PUBLIC_WS_BASE") + or "ws://127.0.0.1:8000" + ).rstrip("/") + return f"{base}/v1/voice-call/ws/{session_id}" + + +# ── HTTP router ──────────────────────────────────────────────────────── + +def build_router(cfg: VoiceCallConfig) -> APIRouter: + """Return the REST sub-router. Empty router if feature off.""" + r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"]) + if not cfg.enabled: + return r # feature off → empty surface + + @r.post("/sessions", response_model=CreateSessionResp, status_code=201) + def create_session( + body: CreateSessionReq, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + user = _resolve_user(authorization, homepilot_session) + try: + row = service.create_session( + user_id=user["id"], + conversation_id=body.conversation_id, + persona_id=body.persona_id, + entry_mode=body.entry_mode, + client_platform=body.device_info.platform, + app_version=body.device_info.app_version, + cfg=cfg, + ) + except PolicyError as exc: + logger.warning( + "[call] create_session policy_error user=%s code=%s status=%s", + user["id"], exc.code, exc.http_status, + ) + return JSONResponse( + status_code=exc.http_status, + content={"ok": False, "code": exc.code, "detail": exc.detail}, + ) + logger.info( + "[call] create_session ok user=%s sid=%s persona=%s conv=%s", + user["id"], row["id"], body.persona_id, body.conversation_id, + ) + # Token is only returned in the create response — never again. + expires_at = int(row["created_at"]) + 15 * 60 * 1000 + return CreateSessionResp( + session_id=row["id"], + ws_url=_ws_url_for(row["id"]), + resume_token=row["resume_token"], + expires_at=expires_at, + max_duration_sec=cfg.max_duration_sec, + capabilities=Capabilities( + interruptions=False, + # barge_in reflects the combined streaming + barge-in + # flag pair — it only works when the session actually + # streams, since there's nothing to cancel in unary. + barge_in=cfg.streaming_enabled and cfg.barge_in_enabled, + # transcript_live = streaming assistant partials. + transcript_live=cfg.streaming_enabled, + streaming=cfg.streaming_enabled, + ), + ) + + @r.get("/sessions/{sid}", response_model=Session) + def get_session( + sid: str, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + user = _resolve_user(authorization, homepilot_session) + row = service.get(sid, user["id"]) + if row is None: + # 404, never 403 — probe-safe. + logger.info("[call] get_session miss user=%s sid=%s", user["id"], sid) + raise HTTPException(status_code=404, detail="Session not found") + return _public_session(row) + + @r.post("/sessions/{sid}/end", response_model=EndSessionResp) + def end_session_ep( + sid: str, + authorization: str = Header(default=""), + homepilot_session: Optional[str] = Cookie(default=None), + ): + user = _resolve_user(authorization, homepilot_session) + row = service.end_session(sid=sid, user_id=user["id"]) + if row is None: + logger.info("[call] end_session miss user=%s sid=%s", user["id"], sid) + raise HTTPException(status_code=404, detail="Session not found") + logger.info( + "[call] end_session ok user=%s sid=%s reason=%s", + user["id"], sid, row.get("ended_reason"), + ) + public = _public_session(row) + return EndSessionResp( + session_id=public.id, + status=public.status, + ended_at=public.ended_at or int(row["updated_at"]), + ended_reason=public.ended_reason or "user_ended", + duration_sec=public.duration_sec or 0, + ) + + @r.get("/hints", response_model=HintsResp) + def hints(): + # Public on purpose — the client needs this BEFORE it has a + # session. Still costs nothing to allow without auth. + return HintsResp() + + return r + + +# ── WS router ────────────────────────────────────────────────────────── + +def build_ws_router(cfg: VoiceCallConfig) -> APIRouter: + r = APIRouter(prefix="/v1/voice-call", tags=["voice-call"]) + if not cfg.enabled or not cfg.websocket_enabled: + return r + + @r.websocket("/ws/{session_id}") + async def voice_call_ws( + websocket: WebSocket, + session_id: str, + resume_token: Optional[str] = Query(default=None), + ): + # Accept first so we can close with a clean error code if needed. + await websocket.accept() + await ws_mod.run_session_ws( + websocket, session_id, resume_token, cfg, + ) + + return r diff --git a/backend/app/voice_call/service.py b/backend/app/voice_call/service.py new file mode 100644 index 0000000000000000000000000000000000000000..85d28b9a25a2aabe27d24cea15a3d23f7f753a1b --- /dev/null +++ b/backend/app/voice_call/service.py @@ -0,0 +1,145 @@ +""" +Pure-Python state machine for a voice-call session. + +No HTTP or WebSocket imports here — the router/WS layers consume this +module. That separation lets the unit tests drive the state machine +end-to-end without spinning up FastAPI. + +State flow (from the design doc, collapsed for MVP): + + connecting → live → ending → ended + │ │ + │ ├─ interrupted (WS dropped, within resume window) + │ │ │ + │ │ └─→ live again (on resume) + │ │ └─→ ended (resume window expired) + │ + └─ ended (ws never opens) +""" +from __future__ import annotations + +import time +from typing import Any, Dict, Optional + +from . import store +from .config import VoiceCallConfig +from .policy import PolicyError, check_session_create + + +_TERMINAL_STATES = {"ended"} +_LIVE_STATES = {"connecting", "live", "interrupted"} + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +# ── create / end ───────────────────────────────────────────────────── + +def create_session( + *, + user_id: str, + conversation_id: Optional[str], + persona_id: Optional[str], + entry_mode: str, + client_platform: Optional[str], + app_version: Optional[str], + cfg: VoiceCallConfig, +) -> Dict[str, Any]: + """Gate through policy, then insert a new session row. + + The returned dict includes the ``resume_token`` which the router + will forward to the client. Callers MUST strip this from any + response that ends up in an observability log. + """ + check_session_create(user_id=user_id, cfg=cfg) + row = store.create_session( + user_id=user_id, + conversation_id=conversation_id, + persona_id=persona_id, + entry_mode=entry_mode, + client_platform=client_platform, + app_version=app_version, + ) + store.append_event( + session_id=row["id"], + seq=0, + event_type="session.created", + payload={"entry_mode": entry_mode}, + ) + return row + + +def end_session( + *, + sid: str, + user_id: str, + reason: str = "user_ended", +) -> Optional[Dict[str, Any]]: + """Mark a session ``ended``. Idempotent: calling twice returns the + same row on the second call. Returns ``None`` if the session doesn't + exist OR doesn't belong to ``user_id`` (probe-safe).""" + row = store.get_session_for_owner(sid, user_id) + if row is None: + return None + if row["status"] in _TERMINAL_STATES: + return row # already ended; idempotent + updated = store.update_session_status(sid, "ended", ended_reason=reason) + if updated is not None: + store.append_event( + session_id=sid, + seq=_next_event_seq(sid), + event_type="session.ended", + payload={"reason": reason}, + ) + return updated + + +def get(sid: str, user_id: str) -> Optional[Dict[str, Any]]: + """Owner-bound getter. Mirrors :func:`store.get_session_for_owner` + so callers don't import store directly.""" + return store.get_session_for_owner(sid, user_id) + + +# ── internal state transitions (called from ws.py) ─────────────────── + +def mark_live(sid: str) -> Optional[Dict[str, Any]]: + """Transition from ``connecting`` → ``live``. Called when the WS + opens with a valid resume token.""" + return store.update_session_status(sid, "live") + + +def mark_interrupted(sid: str) -> Optional[Dict[str, Any]]: + return store.update_session_status(sid, "interrupted") + + +def mark_ended_by_timeout(sid: str, reason: str) -> Optional[Dict[str, Any]]: + return store.update_session_status(sid, "ended", ended_reason=reason) + + +# ── sequence bookkeeping ───────────────────────────────────────────── + +# The server emits events with a monotonically-increasing ``seq``. We +# keep a tiny per-process cache to avoid a SELECT MAX() on every write. +# The cache is seeded on first use by reading the current max seq for +# the session — so it recovers cleanly across process restarts. + +from threading import Lock +_seq_cache: Dict[str, int] = {} +_seq_lock = Lock() + + +def _next_event_seq(session_id: str) -> int: + with _seq_lock: + if session_id in _seq_cache: + _seq_cache[session_id] += 1 + return _seq_cache[session_id] + events = store.list_events(session_id, limit=1000) + current_max = max((e["seq"] for e in events), default=-1) + _seq_cache[session_id] = current_max + 1 + return _seq_cache[session_id] + + +def _reset_for_tests() -> None: + with _seq_lock: + _seq_cache.clear() diff --git a/backend/app/voice_call/store.py b/backend/app/voice_call/store.py new file mode 100644 index 0000000000000000000000000000000000000000..f39a879fdee183283bf655fc6b9178c990e96e08 --- /dev/null +++ b/backend/app/voice_call/store.py @@ -0,0 +1,293 @@ +""" +SQLite access for voice_call. + +Follows the same pattern as backend/app/storage.py and users.py: +direct sqlite3 calls, no ORM, per-call connections. Tables are created +lazily by :func:`ensure_schema` — no separate migration file, no side +effects on import. If the feature is disabled, this module is never +loaded and no tables are created. +""" +from __future__ import annotations + +import json +import secrets +import sqlite3 +import time +from typing import Any, Dict, List, Optional + + +# Re-use the same DB path the rest of HomePilot uses. +def _db_path() -> str: + from ..storage import _get_db_path # lazy import, avoids cycles + return _get_db_path() + + +def _conn() -> sqlite3.Connection: + con = sqlite3.connect(_db_path()) + con.row_factory = sqlite3.Row + return con + + +_SCHEMA_READY = False + + +def ensure_schema() -> None: + """Create voice_call tables if they don't exist. Safe to call repeatedly.""" + global _SCHEMA_READY + if _SCHEMA_READY: + return + con = _conn() + cur = con.cursor() + cur.executescript( + """ + CREATE TABLE IF NOT EXISTS voice_call_sessions ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + conversation_id TEXT, + persona_id TEXT, + entry_mode TEXT NOT NULL, + status TEXT NOT NULL, + resume_token TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER, + ended_reason TEXT, + client_platform TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS ix_vcs_user_started + ON voice_call_sessions (user_id, started_at DESC); + CREATE INDEX IF NOT EXISTS ix_vcs_user_status + ON voice_call_sessions (user_id, status); + + CREATE TABLE IF NOT EXISTS voice_call_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_type TEXT NOT NULL, + event_ts INTEGER NOT NULL, + payload TEXT NOT NULL DEFAULT '{}' + ); + CREATE INDEX IF NOT EXISTS ix_vce_session_seq + ON voice_call_events (session_id, seq); + """ + ) + con.commit() + con.close() + _SCHEMA_READY = True + + +# ── session CRUD ────────────────────────────────────────────────────── + +def new_session_id() -> str: + return "vcs_" + secrets.token_urlsafe(16) + + +def new_resume_token() -> str: + return secrets.token_urlsafe(24) + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def create_session( + *, + user_id: str, + conversation_id: Optional[str], + persona_id: Optional[str], + entry_mode: str, + client_platform: Optional[str], + app_version: Optional[str], +) -> Dict[str, Any]: + """Insert a session row in the ``connecting`` state and return it.""" + ensure_schema() + sid = new_session_id() + token = new_resume_token() + now = _now_ms() + con = _conn() + cur = con.cursor() + cur.execute( + """ + INSERT INTO voice_call_sessions + (id, user_id, conversation_id, persona_id, entry_mode, status, + resume_token, started_at, ended_at, ended_reason, + client_platform, app_version, created_at, updated_at) + VALUES + (?, ?, ?, ?, ?, 'connecting', ?, ?, NULL, NULL, ?, ?, ?, ?) + """, + (sid, user_id, conversation_id, persona_id, entry_mode, token, + now, client_platform, app_version, now, now), + ) + con.commit() + con.close() + return _select_session(sid) # type: ignore[return-value] + + +def _row_to_dict(row: sqlite3.Row) -> Dict[str, Any]: + d = {k: row[k] for k in row.keys()} + return d + + +def _select_session(sid: str) -> Optional[Dict[str, Any]]: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute("SELECT * FROM voice_call_sessions WHERE id = ?", (sid,)) + row = cur.fetchone() + con.close() + return _row_to_dict(row) if row else None + + +def get_session_for_owner(sid: str, user_id: str) -> Optional[Dict[str, Any]]: + """Fetch a session ONLY if it belongs to ``user_id``. + + Returns ``None`` for both "not found" and "wrong owner" so callers + cannot probe session ids they don't own. Matches the pattern used + by studio render jobs. + """ + row = _select_session(sid) + if row and row["user_id"] == user_id: + return row + return None + + +def count_active_sessions_for_user(user_id: str) -> int: + """Count sessions that are still 'live' or 'connecting' or 'interrupted'.""" + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + """ + SELECT COUNT(*) AS n FROM voice_call_sessions + WHERE user_id = ? + AND status IN ('connecting', 'live', 'interrupted') + """, + (user_id,), + ) + n = int(cur.fetchone()["n"]) + con.close() + return n + + +def recent_session_creates_for_user(user_id: str, window_sec: int) -> int: + """Session creates by ``user_id`` in the last ``window_sec`` seconds.""" + ensure_schema() + con = _conn() + cur = con.cursor() + since = _now_ms() - int(window_sec * 1000) + cur.execute( + """ + SELECT COUNT(*) AS n FROM voice_call_sessions + WHERE user_id = ? AND created_at >= ? + """, + (user_id, since), + ) + n = int(cur.fetchone()["n"]) + con.close() + return n + + +def update_session_status( + sid: str, + status: str, + *, + ended_reason: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + ensure_schema() + now = _now_ms() + con = _conn() + cur = con.cursor() + if status == "ended": + cur.execute( + """ + UPDATE voice_call_sessions + SET status = 'ended', ended_at = ?, ended_reason = ?, updated_at = ? + WHERE id = ? + """, + (now, ended_reason or "user_ended", now, sid), + ) + else: + cur.execute( + """ + UPDATE voice_call_sessions + SET status = ?, updated_at = ? + WHERE id = ? + """, + (status, now, sid), + ) + con.commit() + con.close() + return _select_session(sid) + + +# ── events ──────────────────────────────────────────────────────────── + +def append_event( + *, + session_id: str, + seq: int, + event_type: str, + payload: Optional[Dict[str, Any]] = None, +) -> None: + """Append a structured event row. Payload defaults to {} — DO NOT + pass raw audio or full user text here unless privacy flag allows it.""" + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + """ + INSERT INTO voice_call_events (session_id, seq, event_type, event_ts, payload) + VALUES (?, ?, ?, ?, ?) + """, + ( + session_id, + int(seq), + event_type, + _now_ms(), + json.dumps(payload or {}, separators=(",", ":")), + ), + ) + con.commit() + con.close() + + +def list_events(session_id: str, limit: int = 500) -> List[Dict[str, Any]]: + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute( + """ + SELECT session_id, seq, event_type, event_ts, payload + FROM voice_call_events + WHERE session_id = ? + ORDER BY seq ASC + LIMIT ? + """, + (session_id, int(limit)), + ) + rows = cur.fetchall() + con.close() + out = [] + for r in rows: + d = _row_to_dict(r) + try: + d["payload"] = json.loads(d.get("payload") or "{}") + except Exception: + d["payload"] = {} + out.append(d) + return out + + +# ── test helper ─────────────────────────────────────────────────────── + +def _reset_for_tests() -> None: + """DELETE every voice_call row. Only used by pytest fixtures.""" + ensure_schema() + con = _conn() + cur = con.cursor() + cur.execute("DELETE FROM voice_call_events") + cur.execute("DELETE FROM voice_call_sessions") + con.commit() + con.close() diff --git a/backend/app/voice_call/turn.py b/backend/app/voice_call/turn.py new file mode 100644 index 0000000000000000000000000000000000000000..42e9f5f13fc552f240cd9c106c8c7fd3c1c94bf2 --- /dev/null +++ b/backend/app/voice_call/turn.py @@ -0,0 +1,125 @@ +""" +Turn bridge — send a ``transcript.final`` from the client to the +existing ``/v1/chat/completions`` endpoint and get back an assistant +reply (also final, no deltas). + +Implementation note +------------------- +The review confirmed the compat endpoint currently returns 501 on +``stream=true`` (openai_compat_endpoint.py:477), so MVP M1 is final- +only. We invoke the endpoint via an internal loopback HTTP call; that +buys: + + * zero import-time coupling to the chat module, + * the same auth / memory / persona context the normal chat path uses, + * a natural seam for the V2 realtime-provider adapter (swap this + function, everything else stays). + +When the feature flag is off this module is never imported. +""" +from __future__ import annotations + +import os +from typing import Any, Dict, List, Optional + +import httpx + + +def _local_backend_url() -> str: + """Where to reach our own FastAPI server over loopback. Honors an + override for containerized / split-process deployments.""" + return ( + os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL") + or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL") + or "http://127.0.0.1:8000" + ).rstrip("/") + + +DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "30")) + + +async def run_turn( + *, + user_text: str, + model: str, + auth_bearer: Optional[str] = None, + system_prompt: Optional[str] = None, + additional_system: Optional[str] = None, + history: Optional[List[Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, +) -> str: + """Run a single assistant turn via the existing chat endpoint. + + Parameters + ---------- + user_text: + The final STT transcript from the client. + model: + The model id. ``persona:`` / ``personality:`` / + plain ollama model name — whatever the existing chat endpoint + accepts. + auth_bearer: + The caller's bearer token, forwarded verbatim so the chat + endpoint sees the same user identity as the WS session owner. + system_prompt / history: + Optional pre-built context. Left empty the endpoint will attach + its own persona / memory context based on the model. + additional_system: + Optional extra system message appended AFTER ``system_prompt`` + (or on its own if no ``system_prompt`` is provided). Additive + only — used by ``persona_call`` to inject a per-turn phone-call + suffix WITHOUT touching the persona's own system prompt that + the compat endpoint already injects for ``persona:*`` models. + + Returns + ------- + The assistant's reply text (the ``choices[0].message.content`` from + the chat completion response). Raises on non-2xx responses so the + caller can surface an ``error`` event over the WS. + """ + messages: List[Dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + if additional_system: + messages.append({"role": "system", "content": additional_system}) + if history: + messages.extend(history) + messages.append({"role": "user", "content": user_text}) + + headers: Dict[str, str] = {"Content-Type": "application/json"} + if auth_bearer: + headers["Authorization"] = f"Bearer {auth_bearer}" + if extra_headers: + headers.update(extra_headers) + + payload = { + "model": model, + "messages": messages, + # Final-only — must stay False at MVP. The compat endpoint + # returns 501 on streaming requests. + "stream": False, + # Keep response tight for voice. Long prose makes for bad audio. + "max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")), + "temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")), + } + + url = f"{_local_backend_url()}/v1/chat/completions" + async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client: + r = await client.post(url, json=payload, headers=headers) + # Let the caller see the HTTP body on error — useful for debugging. + if r.status_code >= 400: + raise RuntimeError( + f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}" + ) + data = r.json() + try: + return ( + data["choices"][0]["message"]["content"] + if data.get("choices") + else "" + ) + except (KeyError, IndexError, TypeError): + raise RuntimeError( + "chat endpoint returned an unexpected body shape: " + + str(data)[:400] + ) diff --git a/backend/app/voice_call/turn_stream.py b/backend/app/voice_call/turn_stream.py new file mode 100644 index 0000000000000000000000000000000000000000..2a72f6baaf6b243fa5c225c4337a3d02ed3790ff --- /dev/null +++ b/backend/app/voice_call/turn_stream.py @@ -0,0 +1,259 @@ +""" +Streaming turn runner — async-generator peer of :func:`turn.run_turn`. + +Yields assistant text chunks from the chat provider as they arrive so +the WebSocket handler can emit ``assistant.partial`` envelopes without +waiting for the full reply. Drops cleanly on barge-in cancel. + +Contract is documented in +``docs/analysis/voice-call-streaming-design.md`` § 4.1 and § 4.2. + +Two execution paths, selected at runtime: + + 1. **Native streaming** — POST to the compat chat endpoint with + ``stream: true`` and consume the SSE / NDJSON response. This is + the real Phase 2 path and delivers sub-500 ms first-audio when + the provider supports it. + + 2. **Chunked-unary fallback** — if the compat endpoint returns 501 + (current default, see ``turn.py`` comment) OR the provider + doesn't support streaming, we call the unary endpoint, receive + the full reply, and emit it in clause-sized chunks with a tiny + delay so the downstream plumbing (``assistant.partial`` → + ``streamTts.appendDelta``) still exercises correctly. + +The fallback yields no latency improvement — its value is keeping +the envelope contract and the client-side streaming TTS alive even +while the compat endpoint hasn't flipped on streaming. When it does, +this module starts getting real deltas automatically. + +The persona_call suffix, persona resolution, and auth are all handled +by ``ws.py`` / the compat endpoint exactly as today. This module +receives the already-resolved ``user_text`` + ``model`` + headers +and does nothing with the persona layer directly. +""" +from __future__ import annotations + +import asyncio +import json +import os +import re +from typing import AsyncIterator, Dict, List, Optional + +import httpx + +from .barge_in import BargeInToken + + +def _local_backend_url() -> str: + return ( + os.getenv("VOICE_CALL_INTERNAL_BACKEND_URL") + or os.getenv("HOMEPILOT_INTERNAL_BACKEND_URL") + or "http://127.0.0.1:8000" + ).rstrip("/") + + +DEFAULT_TIMEOUT_SEC = float(os.getenv("VOICE_CALL_TURN_TIMEOUT_SEC", "60")) +# Chunk pacing for the fallback path. Kept deliberately small so the +# fallback "feels like" streaming without flooding the client. +_FALLBACK_INTER_CHUNK_MS = int(os.getenv("VOICE_CALL_FALLBACK_CHUNK_MS", "35")) + +# Simple clause splitter used only by the fallback path. Deliberately +# naive: split on sentence-enders first, then on commas if a sentence +# is too long for one chunk. +_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+") +_CHUNK_MAX_CHARS = 90 + + +def _split_for_chunking(text: str) -> List[str]: + if not text: + return [] + parts = [p for p in _SENTENCE_SPLIT.split(text) if p] + out: List[str] = [] + for part in parts: + if len(part) <= _CHUNK_MAX_CHARS: + out.append(part) + continue + # Long sentence — split on commas, rejoining short pieces so + # we don't over-chunk. + comma_bits = [b.strip() for b in part.split(",") if b.strip()] + buf = "" + for bit in comma_bits: + if len(buf) + len(bit) + 2 <= _CHUNK_MAX_CHARS: + buf = f"{buf}, {bit}" if buf else bit + else: + if buf: + out.append(buf) + buf = bit + if buf: + out.append(buf) + # Re-attach trailing whitespace so concatenation preserves spacing. + return [f"{p} " if not p.endswith(" ") else p for p in out] + + +def _extract_delta_from_sse_line(line: str) -> Optional[str]: + """Parse one SSE / NDJSON frame from a streaming chat endpoint. + + Supports both the OpenAI-style ``data: {...}`` SSE format and + Ollama's raw NDJSON (``{"message": {...}}`` per line). + """ + line = line.strip() + if not line: + return None + if line.startswith("data:"): + line = line[5:].strip() + if line == "[DONE]": + return None + try: + obj = json.loads(line) + except json.JSONDecodeError: + return None + # OpenAI streaming: choices[0].delta.content + try: + choices = obj.get("choices") or [] + if choices: + delta = (choices[0] or {}).get("delta") or {} + content = delta.get("content") + if isinstance(content, str) and content: + return content + except (AttributeError, TypeError): + pass + # Ollama streaming: message.content + try: + msg = obj.get("message") or {} + content = msg.get("content") + if isinstance(content, str) and content: + return content + except (AttributeError, TypeError): + pass + return None + + +async def _yield_native_stream( + client: httpx.AsyncClient, + url: str, + payload: Dict, + headers: Dict[str, str], + cancel_token: BargeInToken, +) -> AsyncIterator[str]: + """Native provider streaming path — returns an async iterator of + deltas. Caller handles 501 / 4xx / 5xx by catching the upstream + status before entering this coroutine.""" + async with client.stream("POST", url, json=payload, headers=headers) as r: + if r.status_code == 501: + # Sentinel for "provider doesn't support streaming yet". + # Caller will fall back to chunked-unary. + await r.aread() + raise _StreamingUnsupported() + if r.status_code >= 400: + body = (await r.aread()).decode("utf-8", errors="replace") + raise RuntimeError( + f"chat stream {url} returned {r.status_code}: {body[:400]}" + ) + async for line in r.aiter_lines(): + if cancel_token.is_cancelled(): + return + delta = _extract_delta_from_sse_line(line) + if delta: + yield delta + + +class _StreamingUnsupported(Exception): + """Raised when the provider returns 501 on ``stream: true`` so the + caller can route to the chunked-unary fallback without conflating + it with an ordinary upstream error.""" + + +async def _yield_chunked_unary( + client: httpx.AsyncClient, + url: str, + payload: Dict, + headers: Dict[str, str], + cancel_token: BargeInToken, +) -> AsyncIterator[str]: + """Fallback path — full unary call, then chunk the reply.""" + unary_payload = dict(payload, stream=False) + r = await client.post(url, json=unary_payload, headers=headers) + if r.status_code >= 400: + raise RuntimeError( + f"chat endpoint {url} returned {r.status_code}: {r.text[:400]}" + ) + data = r.json() + try: + reply = ( + data["choices"][0]["message"]["content"] + if data.get("choices") + else "" + ) + except (KeyError, IndexError, TypeError): + raise RuntimeError( + f"chat endpoint returned an unexpected body shape: {str(data)[:400]}" + ) + for chunk in _split_for_chunking(reply): + if cancel_token.is_cancelled(): + return + yield chunk + # Small delay so the client's streaming TTS has a beat to + # synth + play before the next chunk arrives. Keeps the + # fallback from flooding the WS. + await asyncio.sleep(_FALLBACK_INTER_CHUNK_MS / 1000.0) + + +async def run_turn_streaming( + *, + user_text: str, + model: str, + auth_bearer: Optional[str] = None, + system_prompt: Optional[str] = None, + additional_system: Optional[str] = None, + history: Optional[List[Dict[str, str]]] = None, + extra_headers: Optional[Dict[str, str]] = None, + cancel_token: BargeInToken, +) -> AsyncIterator[str]: + """Yield assistant text chunks for the given user turn. + + Exits cleanly (no exception) when ``cancel_token`` is tripped by + a barge-in. The caller is responsible for emitting the final + ``assistant.turn_end`` envelope regardless of exit reason. + """ + messages: List[Dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + if additional_system: + messages.append({"role": "system", "content": additional_system}) + if history: + messages.extend(history) + messages.append({"role": "user", "content": user_text}) + + headers: Dict[str, str] = {"Content-Type": "application/json"} + if auth_bearer: + headers["Authorization"] = f"Bearer {auth_bearer}" + if extra_headers: + headers.update(extra_headers) + + payload = { + "model": model, + "messages": messages, + "stream": True, + "max_tokens": int(os.getenv("VOICE_CALL_TURN_MAX_TOKENS", "300")), + "temperature": float(os.getenv("VOICE_CALL_TURN_TEMPERATURE", "0.7")), + } + + url = f"{_local_backend_url()}/v1/chat/completions" + + async with httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_SEC) as client: + # Try native streaming first. + try: + async for delta in _yield_native_stream( + client, url, payload, headers, cancel_token, + ): + yield delta + return + except _StreamingUnsupported: + # Compat endpoint currently returns 501 on stream=true; + # fall back to unary + chunker. Covered by test. + pass + async for delta in _yield_chunked_unary( + client, url, payload, headers, cancel_token, + ): + yield delta diff --git a/backend/app/voice_call/ws.py b/backend/app/voice_call/ws.py new file mode 100644 index 0000000000000000000000000000000000000000..814eae1c1a08fc5ad0b06a98cd8e94f48fdce825 --- /dev/null +++ b/backend/app/voice_call/ws.py @@ -0,0 +1,605 @@ +""" +WebSocket orchestrator for voice-call sessions. + +Transport +--------- +Path: ``/v1/voice-call/ws/{session_id}`` +Query param: ``resume_token=`` + (headers don't round-trip cleanly from browsers; a + short-lived query token is the standard WS-auth trick) + +Protocol +-------- +Server and client exchange JSON envelopes:: + + { "type": "", "seq": , "ts": , "payload": { ... } } + +Client → server (each is harmless if the server can't handle it — +forward-compatible by design): + ui.state { muted, speaker_on, backgrounded } + transcript.final { text, lang? } + call.control { action: "end" | "ping" } + +Server → client: + call.state { status, since } + transcript.final { role: "assistant", text } + error { code, message } + pong { ts } + safety.notice (forwarded from upstream providers, reserved) + +Out of scope for MVP (per review): transcript.partial (requires chat +endpoint streaming, which returns 501 today), raw audio frames, server- +side barge-in. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from typing import Any, Dict, Optional + +from fastapi import WebSocket, WebSocketDisconnect, status + +from . import barge_in as _barge_in +from . import service, store, turn, turn_stream +from .config import VoiceCallConfig + + +logger = logging.getLogger("voice_call.ws") + + +def _clog(event: str, **kwargs: Any) -> None: + """Structured ``[call]`` log line — mirrors the frontend's clog() + format so a single grep surfaces the whole round-trip. One + ``logger.info`` per inflection point; fields stay flat so + downstream log-shippers don't have to deserialize. + + Used at: session accept (open), call.state live (ready), opener + greeting (greet), user transcript in (turn_user_in), assistant + reply out (turn_assistant_out), cooperative cancel (barge_in), + user-ended (hangup), WS teardown (close).""" + logger.info("[call] %s %s", event, json.dumps(kwargs, separators=(",", ":"), default=str)) + +HEARTBEAT_INTERVAL_SEC = 10.0 + +# Protocol sentinels — reserved strings the frontend may send as a +# ``transcript.final`` payload to nudge backend state without asking +# the LLM to react. We drop these on the floor before the turn runner +# sees them; otherwise the model treats them as literal user speech +# and produces a confused reply (the classic "I'm not sure what +# [phone-call-open] means" double-greet). Add new sentinels here only +# when both sides have landed. +_PROTOCOL_SENTINELS = frozenset({ + "[phone-call-open]", + "[phone-call-close]", +}) + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +async def _send(ws: WebSocket, type_: str, payload: Dict[str, Any], *, + session_id: str) -> None: + """Send a server→client event with a monotonic seq number.""" + seq = service._next_event_seq(session_id) + env = {"type": type_, "seq": seq, "ts": _now_ms(), "payload": payload} + try: + await ws.send_text(json.dumps(env, separators=(",", ":"))) + except Exception: + # WebSocket already closed; caller will clean up. + raise + # Only persist the *type* for privacy unless the user has explicitly + # opted into transcript storage. + store.append_event( + session_id=session_id, seq=seq, event_type=type_, + payload={}, # intentionally empty + ) + + +async def _send_error(ws: WebSocket, session_id: str, code: str, msg: str) -> None: + try: + await _send(ws, "error", + {"code": code, "message": msg}, + session_id=session_id) + except Exception: + pass # best-effort + + +async def _resolve_session_for_ws( + ws: WebSocket, session_id: str, resume_token: Optional[str], + cfg: VoiceCallConfig, +) -> Optional[Dict[str, Any]]: + """Validate the handshake. Closes the socket with a clean code if + anything is off.""" + if not cfg.websocket_enabled: + await ws.close(code=status.WS_1008_POLICY_VIOLATION, + reason="websocket-disabled") + return None + row = store._select_session(session_id) + if row is None: + await ws.close(code=status.WS_1008_POLICY_VIOLATION, + reason="session-not-found") + return None + if not resume_token or row.get("resume_token") != resume_token: + await ws.close(code=status.WS_1008_POLICY_VIOLATION, + reason="bad-resume-token") + return None + if row.get("status") == "ended": + await ws.close(code=status.WS_1008_POLICY_VIOLATION, + reason="session-ended") + return None + # Resume window: an "interrupted" session is only acceptable if the + # drop happened within the configured window. + if row.get("status") == "interrupted": + last_update = int(row.get("updated_at") or 0) + if _now_ms() - last_update > cfg.resume_window_sec * 1000: + service.mark_ended_by_timeout(session_id, "resume_expired") + await ws.close(code=status.WS_1008_POLICY_VIOLATION, + reason="resume-expired") + return None + return row + + +async def run_session_ws( + ws: WebSocket, + session_id: str, + resume_token: Optional[str], + cfg: VoiceCallConfig, +) -> None: + """Main WS loop for one session. Called from router after + ``await ws.accept()``. + + Ownership is established by the resume token alone (token is only + ever handed to the user who created the session). The per-turn + chat request also carries the caller's bearer — so the chat + endpoint does its own auth and refuses foreign personas even if + the token somehow leaks.""" + session = await _resolve_session_for_ws(ws, session_id, resume_token, cfg) + if session is None: + return # close already issued + + # Extract the bearer from the WS query so turn.py can re-use it. + # Browsers can't attach custom headers to WS; the client passes + # ?token= alongside ?resume_token=. + query_bearer = ws.query_params.get("token") or None + + sid = session["id"] + _clog("lifecycle_open", sid=sid, persona_id=session.get("persona_id")) + service.mark_live(sid) + await _send(ws, "call.state", {"status": "live"}, session_id=sid) + _clog("lifecycle_ready", sid=sid) + + # ── "Theory of answering" hook ──────────────────────────────── + # When persona_call is on, the persona answers first — the way a + # real callee picks up the phone — instead of sitting silent + # until the caller speaks. The opening text comes from the + # curated bank in persona_call/openings.py; selection is + # deterministic (seeded by session_id) + persona-aware + rotated + # against the anti-repetition ledger, so the same caller never + # hears the same greeting twice in a row. + try: + from ..persona_call import ( + config as _pc_cfg_mod, + context as _pc_context, + facets as _pc_facets, + openings as _pc_openings, + store as _pc_store, + ) + _pc_cfg = _pc_cfg_mod.load() + if _pc_cfg.enabled: + _pc_store.ensure_schema() + _pc_store.ensure_state(sid) + _state = _pc_store.get_state(sid) or {} + forbidden = list(_state.get("recent_openers") or []) + _pc_env = _pc_context.compute_env(tz=None, weeks_since_last_call=-1) + _pc_facets_obj = _pc_facets.for_persona_id(session.get("persona_id")) + # We need the template *id* too so we can push it into + # the ledger, so call choose() (not render() directly). + tpl = _pc_openings.choose( + _pc_facets_obj, _pc_env, + session_id=sid, + turn_index=1, + forbidden_ids=forbidden, + ) + greeting_text = None + if tpl is not None: + greeting_text = _pc_openings._interpolate( + tpl.text, + # Fall back to "Assistant" only if the session row + # has no persona_id — matches the frontend label. + (session.get("persona_id") or "Assistant").replace("persona:", "").strip(), + ) + # Append the chosen id to the opener ledger, keeping + # only the last N so the ledger doesn't grow + # unbounded across long-lived sessions. + window = max(1, _pc_cfg.opener_ledger_window) + new_openers = (list(forbidden) + [tpl.id])[-window:] + _pc_store.update_state(sid, recent_openers=new_openers) + if greeting_text: + await _send(ws, "transcript.final", + {"role": "assistant", "text": greeting_text}, + session_id=sid) + _clog("lifecycle_greet", sid=sid, text_len=len(greeting_text)) + except Exception as _pc_err: + # Shadow-fail — the call still works without a greeting. + logger.warning("[persona_call] opening greeting skipped: %s", _pc_err) + + # Heartbeat keeps idle TCP paths alive on mobile networks. + hb_task = asyncio.create_task(_heartbeat_loop(ws, sid, cfg)) + + try: + while True: + try: + text = await asyncio.wait_for( + ws.receive_text(), + timeout=cfg.idle_timeout_sec, + ) + except asyncio.TimeoutError: + await _send_error(ws, sid, "idle_timeout", + "No activity for a while — ending call.") + service.end_session(sid=sid, user_id=session["user_id"], + reason="idle") + await _send(ws, "call.state", {"status": "ended", + "reason": "idle"}, + session_id=sid) + break + + try: + env = json.loads(text) + except json.JSONDecodeError: + await _send_error(ws, sid, "bad_json", "Invalid JSON envelope.") + continue + + evt_type = env.get("type") + payload = env.get("payload") or {} + + if evt_type == "call.control": + action = (payload.get("action") or "").strip() + if action == "end": + service.end_session(sid=sid, user_id=session["user_id"], + reason="user_ended") + await _send(ws, "call.state", {"status": "ended", + "reason": "user_ended"}, + session_id=sid) + _clog("lifecycle_hangup", sid=sid, reason="user_ended") + break + if action == "ping": + await _send(ws, "pong", {}, session_id=sid) + + elif evt_type == "ui.state": + # Pure bookkeeping. Client tells us muted / speaker_on / + # backgrounded; we persist only the type (privacy). + pass + + elif evt_type == "transcript.final": + text_in = (payload.get("text") or "").strip() + if not text_in: + continue + # Protocol sentinels are NOT user utterances — they + # exist solely so the frontend can signal state + # transitions (phase machine nudges, etc.) without + # the LLM reacting to the literal string. Drop and + # continue; the phase machine already advances on the + # first real user utterance, so no special handling + # is required here beyond not running the turn. + if text_in in _PROTOCOL_SENTINELS: + continue + _clog("turn_user_in", sid=sid, text_len=len(text_in)) + model = ( + payload.get("model") + or session.get("persona_id") + and f"persona:{session['persona_id']}" + or "llama3:8b" + ) + + # ── persona_call hook ─────────────────────────────── + # Additive, behind its own feature flag. Composes a + # per-turn system SUFFIX (never touches the persona's + # base system prompt) and — if cfg.apply is True — + # appends it as an extra system message on the chat + # call. Wraps the turn in a filler scheduler so a long + # LLM wait produces a "hmm…" event within 600 ms. + pc_additional_system: Optional[str] = None + pc_persona_id: Optional[str] = session.get("persona_id") + try: + from ..persona_call import ( + config as _pc_cfg_mod, + context as _pc_context, + directive as _pc_directive, + facets as _pc_facets, + latency as _pc_latency, + ) + _pc_cfg = _pc_cfg_mod.load() + except Exception: + _pc_cfg = None # persona_call not importable; skip + + _pc_facets_obj = None + if _pc_cfg is not None and _pc_cfg.enabled: + try: + _pc_env = _pc_context.compute_env( + tz=(session.get("client_platform") and None), + weeks_since_last_call=-1, + ) + _pc_facets_obj = _pc_facets.for_persona_id(pc_persona_id) + composed = _pc_directive.compose( + session_id=sid, + persona_id=pc_persona_id, + user_text=text_in, + env=_pc_env, + cfg=_pc_cfg, + facets=_pc_facets_obj, + ) + if _pc_cfg.apply and composed.system_suffix: + pc_additional_system = composed.system_suffix + except Exception as _pc_err: + # Shadow fail — turn still runs without the + # phone-call suffix. persona_call must never + # break an otherwise-working call. + logger.warning( + "[persona_call] compose skipped: %s", _pc_err + ) + + async def _pc_send(env_: dict) -> None: + try: + await ws.send_text( + json.dumps(env_, separators=(",", ":")) + ) + except Exception: + pass + + # ── Streaming vs unary turn — additive branch ────── + # When VOICE_CALL_STREAMING_ENABLED=true the turn runs + # through turn_stream.run_turn_streaming and emits + # ``assistant.partial`` + ``assistant.turn_end`` envelopes + # so the client's streaming TTS can begin speaking long + # before the full reply has arrived. When the flag is + # off we take the original path byte-for-byte. + if cfg.streaming_enabled: + turn_id = f"t_{uuid.uuid4().hex[:12]}" + token = _barge_in.new_token(sid, turn_id) + assistant_text_parts: list[str] = [] + end_reason = "complete" + try: + if ( + _pc_cfg is not None + and _pc_cfg.enabled + and _pc_facets_obj is not None + ): + # Filler scheduler still wraps streaming — + # if the FIRST delta takes > N ms the + # client still gets a ``hmm…`` event. + async with _pc_latency.FillerScheduler( + send=_pc_send, + facets=_pc_facets_obj, + cfg=_pc_cfg, + session_id=sid, + ): + index = 0 + async for delta in turn_stream.run_turn_streaming( + user_text=text_in, + model=model, + auth_bearer=query_bearer, + additional_system=pc_additional_system, + cancel_token=token, + ): + if token.is_cancelled(): + break + assistant_text_parts.append(delta) + await _send( + ws, "assistant.partial", + { + "turn_id": turn_id, + "delta": delta, + "index": index, + }, + session_id=sid, + ) + index += 1 + else: + index = 0 + async for delta in turn_stream.run_turn_streaming( + user_text=text_in, + model=model, + auth_bearer=query_bearer, + additional_system=pc_additional_system, + cancel_token=token, + ): + if token.is_cancelled(): + break + assistant_text_parts.append(delta) + await _send( + ws, "assistant.partial", + { + "turn_id": turn_id, + "delta": delta, + "index": index, + }, + session_id=sid, + ) + index += 1 + if token.is_cancelled(): + end_reason = "cancelled" + except Exception as exc: + logger.warning("[voice_call] stream turn failed: %s", exc) + end_reason = "error" + finally: + # Always emit turn_end so the client can close + # the turn bookkeeping cleanly. + _barge_in.clear_session(sid) + assistant_text = "".join(assistant_text_parts) + await _send( + ws, "assistant.turn_end", + { + "turn_id": turn_id, + "reason": end_reason, + "full_text": assistant_text, + }, + session_id=sid, + ) + _clog( + "turn_assistant_out", + sid=sid, + turn_id=turn_id, + reason=end_reason, + text_len=len(assistant_text), + route="stream", + ) + else: + try: + if ( + _pc_cfg is not None + and _pc_cfg.enabled + and _pc_facets_obj is not None + ): + async with _pc_latency.FillerScheduler( + send=_pc_send, + facets=_pc_facets_obj, + cfg=_pc_cfg, + session_id=sid, + ): + assistant_text = await turn.run_turn( + user_text=text_in, + model=model, + auth_bearer=query_bearer, + additional_system=pc_additional_system, + ) + else: + assistant_text = await turn.run_turn( + user_text=text_in, + model=model, + auth_bearer=query_bearer, + additional_system=pc_additional_system, + ) + except Exception as exc: + logger.warning("[voice_call] turn failed: %s", exc) + await _send_error(ws, sid, "turn_failed", str(exc)[:400]) + continue + + await _send(ws, "transcript.final", + {"role": "assistant", "text": assistant_text}, + session_id=sid) + _clog( + "turn_assistant_out", + sid=sid, + text_len=len(assistant_text), + route="unary", + ) + + # Record the reply into the anti-repetition ledger so + # the NEXT turn can forbid the just-used opener/ack. + if _pc_cfg is not None and _pc_cfg.enabled: + try: + _pc_directive.record_persona_reply( + session_id=sid, + persona_id=pc_persona_id, + reply=assistant_text, + cfg=_pc_cfg, + ) + except Exception as _pc_err: + logger.warning( + "[persona_call] record_reply skipped: %s", + _pc_err, + ) + + elif evt_type == "user.barge_in": + # Phase 3 — cooperative cancel. Client's VAD detected + # user speech-start while the persona was mid-reply. + # § 4.4 of the streaming design doc. + if not (cfg.streaming_enabled and cfg.barge_in_enabled): + continue + turn_id = (payload.get("turn_id") or "").strip() + if not turn_id: + continue + cancelled = _barge_in.cancel_active(sid, turn_id) + if cancelled: + # Emit the ack IMMEDIATELY — client stops TTS on + # receipt, shaving ~50-100 ms off the perceived + # barge-in latency vs waiting for the generator + # to notice the event on its next poll. + await _send( + ws, "assistant.cancel", + {"turn_id": turn_id, "cause": "user_barge_in"}, + session_id=sid, + ) + _clog("barge_in", sid=sid, turn_id=turn_id, cause="user_barge_in") + # Else: stale turn_id (turn already ended). Silent drop. + + elif evt_type == "transcript.partial": + # Secondary barge-in signal — if the client's explicit + # user.barge_in didn't arrive but the STT started + # decoding words, that also means the user spoke. + # Only fires while a turn is active. + if not (cfg.streaming_enabled and cfg.barge_in_enabled): + continue + active = _barge_in.get_active(sid) + if active is not None and not active.is_cancelled(): + cancelled = _barge_in.cancel_active(sid, active.turn_id) + if cancelled: + await _send( + ws, "assistant.cancel", + { + "turn_id": active.turn_id, + "cause": "user_partial", + }, + session_id=sid, + ) + + else: + # Forward-compat: unknown event type is a no-op, not an + # error. Older servers + newer clients stay friendly. + pass + except WebSocketDisconnect: + # Client dropped — park the session as 'interrupted' so a resume + # request within the window can pick it back up. + _barge_in.clear_session(sid) + service.mark_interrupted(sid) + except Exception as exc: + logger.exception("[voice_call] ws error for %s: %s", sid, exc) + try: + await _send_error(ws, sid, "internal", "Internal error") + except Exception: + pass + _barge_in.clear_session(sid) + service.mark_interrupted(sid) + finally: + _clog("lifecycle_close", sid=sid) + _barge_in.clear_session(sid) + hb_task.cancel() + try: + await hb_task + except Exception: + pass + + +async def _heartbeat_loop(ws: WebSocket, session_id: str, cfg: VoiceCallConfig) -> None: + """Send a server-initiated ping every N seconds. Doubles as a + liveness check + hard-duration-cap enforcement.""" + while True: + await asyncio.sleep(HEARTBEAT_INTERVAL_SEC) + row = store._select_session(session_id) + if row is None or row["status"] == "ended": + return + # Enforce hard cap. + started = int(row.get("started_at") or 0) + if started and (_now_ms() - started) >= cfg.max_duration_sec * 1000: + try: + await _send(ws, "call.state", + {"status": "ending", "reason": "max_duration"}, + session_id=session_id) + except Exception: + return + service.end_session(sid=session_id, user_id=row["user_id"], + reason="max_duration") + try: + await ws.close(code=status.WS_1000_NORMAL_CLOSURE, + reason="max-duration") + except Exception: + pass + return + try: + await _send(ws, "ping", {}, session_id=session_id) + except Exception: + return diff --git a/frontend/.cache-bust b/frontend/.cache-bust index ce828f0b591260e967fd3409379481887a216a98..7061be1b2f147799b7e23b4f320b8c58ad41d604 100644 --- a/frontend/.cache-bust +++ b/frontend/.cache-bust @@ -1 +1 @@ -1776444513 +1776516992 diff --git a/frontend/src/ui/App.tsx b/frontend/src/ui/App.tsx index 76a391bd68c6b7972d5d3aeeddcf8c3045fa73fe..1987243ed4b94abc5b23da8d85296e902c1f5e96 100644 --- a/frontend/src/ui/App.tsx +++ b/frontend/src/ui/App.tsx @@ -21,6 +21,7 @@ import { Copy, RotateCw, PenLine, + Phone, Users, EyeOff, BookOpen, @@ -34,6 +35,9 @@ import UserAvatar from './components/UserAvatar' import AccountMenu, { type AccountMenuUser } from './components/AccountMenu' import { useAuth } from './components/AuthGate' import VoiceMode, { stripMarkdownForSpeech } from './VoiceModeGrok' +import CallOverlay from './CallOverlay' +import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log' +import PostCallCard from './phone/PostCallCard' // Legacy voice mode available as: import VoiceModeLegacy from './VoiceModeLegacy' import ProjectsView from './ProjectsView' import ImagineView from './Imagine' @@ -113,6 +117,22 @@ export type Msg = { intent: 'generate_images' | 'generate_videos' prompt: string } + // "Call memory card" — when present the message renders as an + // inline record of a just-ended voice call instead of a regular + // bubble. Additive: untouched by every existing code path. + callMemory?: { + durationSec: number + /** Epoch-ms timestamp of when the call ended. Rendered as an + * auditable stamp (e.g. "7:36 PM") in the card header. */ + endedAt?: number + /** Persona the user was speaking with during this call — feeds + * PostCallCard's "Call with X" / "X remembers" headers. */ + personaName?: string + /** Optional transcript the user can expand inline from the card. + * Populated by App.tsx at call-end by slicing chatMessages + * between the call's open + close timestamps. */ + transcript?: Array<{ who: 'user' | 'assistant'; text: string }> + } } // --------------------------------------------------------------------------- @@ -1574,12 +1594,92 @@ function useCopyMessage(timeoutMs = 900) { return { copied, copy } } +/** + * CallMemoryCard — enterprise-style inline record of a just-ended + * call. Rendered inside the chat stream as a neutral system event + * rather than a floating toast or an emotional "moment" card. The + * tone is auditable, not expressive: the card states what happened, + * when, and offers one standard system action to resume. + */ +function CallMemoryCard({ + durationSec, + endedAt, + onSpeakAgain, +}: { + durationSec: number + endedAt?: number + onSpeakAgain?: () => void +}) { + const fmt = (s: number) => { + if (s < 60) return `${s}s` + const m = Math.floor(s / 60) + const r = s % 60 + return r === 0 ? `${m}m` : `${m}m ${r}s` + } + const stamp = endedAt + ? new Date(endedAt).toLocaleTimeString(undefined, { + hour: 'numeric', minute: '2-digit', + }) + : '' + return ( +
+
+ +
+
+ {/* Meta row — factual header, low emphasis */} +
+ Call + · + + {fmt(durationSec)} + + {stamp ? ( + <> + · + {stamp} + + ) : null} +
+ {/* Body — neutral factual line, no emotional copy */} +
+ Voice session completed +
+ {/* CTA — system-style outline button, same weight as header icons */} +
+ +
+
+
+ ) +} + + function ChatState({ messages, setLightbox, endRef, mode, onNewConversation, + onStartCall, onRetryMessage, chatSettings, onUpdateChatSettings, @@ -1598,6 +1698,9 @@ function ChatState({ endRef: React.RefObject mode: Mode onNewConversation: () => void + /** Primary call action — opens the live-voice mode. Optional so + * callers that don't want the button can omit it. */ + onStartCall?: () => void onRetryMessage: (id: string) => void chatSettings: ChatScopedSettings onUpdateChatSettings: (next: ChatScopedSettings) => void @@ -1613,6 +1716,9 @@ function ChatState({ }) { const { copied, copy } = useCopyMessage() const [chatSettingsOpen, setChatSettingsOpen] = useState(false) + const handleStartCall = useCallback(() => { + onStartCall?.() + }, [onStartCall]) // Local overrides for view-pack angle switching (keyed by message id) const [angleOverrides, setAngleOverrides] = useState>({}) @@ -1650,9 +1756,24 @@ function ChatState({ return (
- {/* Top-right fixed: Chat settings gear + New Chat icon (Phase 2, additive) */} + {/* Top-right fixed: a single flat group of neutral header icons. + The call button is intentionally *not* a CTA — it sits inline + with the other utility icons and only opens a dedicated call + overlay (see CallOverlay). Call mode is a distinct session, + not a styling of the header. */}
-
+
+ {onStartCall && ( + + )}
))}
@@ -2040,6 +2176,22 @@ export default function App() { }) const [voiceConversationId, setVoiceConversationId] = useState(() => uuid()) const [lightbox, setLightbox] = useState(null) + // Call mode — a dedicated session distinct from Voice mode. The + // header 📞 opens a centered CallOverlay; it does not swap modes. + const [callOpen, setCallOpen] = useState(false) + // `skipDialing` is flipped on for the "Speak again" re-entry so the + // user doesn't have to sit through "calling…" twice in a row. + const [callSkipDialing, setCallSkipDialing] = useState(false) + // Snapshot of chatMessages.length at the moment a call opens — + // used at call-end to slice the transcript of turns that happened + // during the call, for PostCallCard's expandable transcript. + const callOpenMsgIndexRef = useRef(-1) + useEffect(() => { + if (callOpen) { + callOpenMsgIndexRef.current = chatMessages.length + } + }, [callOpen]) // intentionally miss chatMessages — we want the + // snapshot at OPEN time, not every keystroke after // Persona Integration — "Save as Persona Avatar" from AvatarStudio (additive) const [saveAsPersonaData, setSaveAsPersonaData] = useState<{ item: GalleryItem; outfits: GalleryItem[]; batchSiblings?: GalleryItem[] } | null>(null) @@ -2311,6 +2463,26 @@ export default function App() { // Track last spoken message to avoid re-speaking const lastSpokenMessageIdRef = useRef(null) + // Call-open history guard — when the overlay opens, seed the + // last-spoken pointer to the CURRENT tail of ``messages``. Without + // this the TTS-on-assistant-reply effect treats the previous + // chat-history tail as a fresh reply the moment ``callOpen`` flips + // true, producing a pre-existing message read-aloud that collides + // with CallOverlay's own fallback opener (the "two openings + // overlapping" symptom). Only NEW assistant messages that land + // DURING the call will be spoken by App.tsx from here on; the + // call-itself-opener is owned by CallOverlay via speakOwned(). + useEffect(() => { + if (!callOpen) return + const tail = messages[messages.length - 1] + lastSpokenMessageIdRef.current = tail ? tail.id : null + // Intentionally single-shot on callOpen flip — we don't want this + // to re-run on every message mutation during the call, otherwise + // the guard would swallow the very replies it's supposed to let + // through. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [callOpen]) + // When a voice session is explicitly chosen from Session Hub, skip the auto-link resolve // to prevent the useEffect from overwriting the chosen session with the active one. const voiceSessionExplicitRef = useRef(false) @@ -3985,10 +4157,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. [authHeaders, conversationId, messages, settings.backendUrl, settings.funMode, settingsDraft] ) - // TTS for assistant responses (speak-once pattern) - ONLY IN VOICE MODE + // TTS for assistant responses (speak-once pattern). Fires when + // the user is in Voice mode OR the CallOverlay is open. The + // second branch matters in the chat-REST fallback path: when + // VOICE_CALL_ENABLED is false server-side, the call runs through + // the regular chat pipeline, assistant replies land in the + // current ``messages`` array, and without this nothing speaks + // them — the user hears silence during the call. + // + // SpeechService.speak + stripMarkdownForSpeech are the canonical + // app-level TTS primitives; reusing them here keeps call-mode + // TTS consistent with voice-mode TTS (same voice pick, same + // markdown sanitation, same single-utterance dedupe via + // lastSpokenMessageIdRef). useEffect(() => { - // Only enable TTS when in Voice mode - if (mode !== 'voice') return + if (mode !== 'voice' && !callOpen) return const ttsEnabled = settingsDraft.ttsEnabled ?? true if (!ttsEnabled || !window.SpeechService) return @@ -4001,6 +4184,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. lastMessage.role === 'assistant' && !lastMessage.pending && lastMessage.text && + !lastMessage.callMemory && lastMessage.id !== lastSpokenMessageIdRef.current ) { // Mark this message as spoken @@ -4009,9 +4193,21 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. // Speak the assistant's response — strip markdown images/links so TTS // doesn't read raw URLs aloud. Images are shown visually instead. const speechText = stripMarkdownForSpeech(lastMessage.text) - if (speechText) window.SpeechService.speak(speechText) + if (!speechText) return + // During a call, route through the shared speakOwned lock so + // this app-level path can't double-speak alongside the overlay's + // fallback opener (both targeting the same assistant message). + // Outside a call, keep the legacy SpeechService.speak path — + // voice-mode has no peer TTS source so the lock would add noise + // without benefit. + if (callOpen && isCallFullDuplexEnabled()) { + clog({ e: 'turn', action: 'assistant_in', route: 'chat_rest' }) + speakOwned('app-level', speechText) + } else { + window.SpeechService.speak(speechText) + } } - }, [messages, settingsDraft.ttsEnabled, mode]) + }, [messages, settingsDraft.ttsEnabled, mode, callOpen]) const uploadAndSend = useCallback( async (file: File) => { @@ -4489,7 +4685,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. {/* Top-right Project Indicator - Only shown in chat mode when a project is active */} {mode === 'chat' && ( -
+
{/* Project Indicator */} {(() => { const currentProjectId = localStorage.getItem('homepilot_current_project') @@ -5049,6 +5245,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. endRef={endRef} mode={mode} onNewConversation={onNewConversation} + onStartCall={() => setCallOpen(true)} onRetryMessage={retryFailedMessage} chatSettings={chatSettings} onUpdateChatSettings={updateChatSettings} @@ -5124,6 +5321,7 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. endRef={endRef} mode={mode} onNewConversation={onNewConversation} + onStartCall={() => setCallOpen(true)} onRetryMessage={retryFailedMessage} chatSettings={chatSettings} onUpdateChatSettings={updateChatSettings} @@ -5149,6 +5347,91 @@ ${personalityPrompt || 'You are a friendly voice assistant. Be helpful and warm. /> ) : null} + {/* Call overlay — a distinct session layered over the chat. + Open it with the 📞 header icon; end it with the red button + inside the overlay. Not the same as Voice mode. */} + { + setCallOpen(false) + // Reset the skip flag once the overlay has closed so the + // next fresh tap on 📞 starts with the full dial phase. + setCallSkipDialing(false) + }} + skipDialing={callSkipDialing} + onEnded={(durationSec) => { + // Append an inline PostCallCard to the chat stream. Captures + // the slice of turns exchanged during the call — which the + // user can expand inline from the card (no modal) — plus + // the persona name + end-of-call timestamp. + const endedAt = Date.now() + const personaName = currentProject?.name || 'Assistant' + const startIdx = callOpenMsgIndexRef.current + // Build a transcript only when we have a valid open-time + // snapshot AND new messages landed during the call. + let transcript: + | Array<{ who: 'user' | 'assistant'; text: string }> + | undefined + if (startIdx >= 0 && startIdx <= chatMessages.length) { + const slice = chatMessages.slice(startIdx) + const lines = slice + .filter((m) => !m.callMemory && (m.text || '').trim()) + .map((m) => ({ + who: (m.role === 'user' ? 'user' : 'assistant') as + | 'user' | 'assistant', + text: m.text.trim(), + })) + if (lines.length > 0) transcript = lines + } + setChatMessages((prev) => [ + ...prev, + { + id: `call-memory-${endedAt}`, + role: 'assistant', + text: '', + callMemory: { + durationSec, + endedAt, + personaName, + transcript, + }, + }, + ]) + }} + personaName={currentProject?.name || 'Assistant'} + messages={chatMessages} + avatarUrl={(() => { + // Resolve the persona's thumbnail the same way + // PersonaSettingsPanel does: prefer + // persona_appearance.selected_thumb_filename, then + // .selected_filename, wrapped in the /files/ auth URL. + const pap = (currentProject?.persona_appearance as Record | undefined) || {} + const rel = + (pap.selected_thumb_filename as string | undefined) || + (pap.selected_filename as string | undefined) + if (!rel) return null + const base = settings.backendUrl.replace(/\/+$/, '') + const tok = localStorage.getItem('homepilot_auth_token') || '' + const clean = rel.replace(/^\/+/, '') + return `${base}/files/${clean}${tok ? `?token=${encodeURIComponent(tok)}` : ''}` + })()} + onSendText={(text) => sendTextOrIntent(text)} + backend={{ + backendUrl: settings.backendUrl, + authToken: localStorage.getItem('homepilot_auth_token'), + conversationId: chatConversationId, + personaId: currentProject?.persona_agent + ? (currentProject.persona_agent.id as string | undefined) || null + : null, + }} + /> + + {/* Previous floating "call ended" toast has been replaced by an + inline CallMemoryCard rendered directly in the chat stream + (see the messages.map branch on m.callMemory). Keeping this + comment as a breadcrumb for reviewers. */} + + {/* Save as Persona Avatar modal (from AvatarStudio gallery) */} {saveAsPersonaData && ( components + * controls.jsx → + * avatar.jsx → + * waveform.jsx → + * call-modal.jsx → + the ModalPresentation wrapper + * + * Layered as a distinct "call mode" on top of the chat — it is NOT + * the same thing as Voice mode: + * 🎤 Voice = input method inside the chat + * 📞 Call = a separate immersive session (this component) + * + * States (mirrors the designer's set): + * connecting · listening · thinking · speaking · muted + * (ended is intentionally short-lived — the overlay fades out.) + * + * This pass wires the visual shell. Real STT/TTS plumbing from + * useVoiceController maps `listening/thinking/speaking` later. + */ +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useVoiceController, type VoiceState } from './voice/useVoiceController' +import { useCallSession } from './call/useCallSession' +import Waveform, { type WaveformMode } from './phone/primitives/Waveform' +import Aura from './phone/primitives/Aura' +import { createStreamingTts, type StreamingTts } from './call/streamTts' +import { clog, speakOwned, isCallFullDuplexEnabled } from './call/log' +import { useBargeInDetector } from './call/bargeIn' + +export type CallState = + | 'dialing' + | 'connecting' + | 'listening' + | 'thinking' + | 'speaking' + | 'muted' + | 'ended' + +// ── Staged lifecycle timings. Numbers chosen so the user *feels* the +// handshake instead of getting a hard screen flip: dial (ringing) → +// connecting → listening (live). Dial is long enough for ~one full +// PSTN ringback cadence (2 s on / 4 s off in the US) so the user +// actually hears the phone "ring" before pickup. +const DIAL_MS = 2200 // 📞 "calling…" — pulse-ring phase + one PSTN ring +const CONNECT_MS = 500 // 🔵 "connecting…" — dots-pulse phase +const END_FADE_MS = 220 // 🔴 end button → modal fade-out → onClose + +// ── Design tokens (ported from phone/tokens.jsx) ────────────────── + +const HP_CALL = { + font: 'Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif', + fontTabular: 'ui-monospace, SFMono-Regular, "Roboto Mono", Menlo, monospace', + backdrop: 'rgba(5, 5, 6, 0.72)', + surface: '#0b0b0c', + surface2: '#121214', + border: 'rgba(255, 255, 255, 0.08)', + text: 'rgba(255, 255, 255, 0.92)', + text2: 'rgba(255, 255, 255, 0.55)', + text3: 'rgba(255, 255, 255, 0.35)', + accent: '#22d3ee', + stateListening: '#22d3ee', + stateThinking: '#a78bfa', + stateSpeaking: '#10b981', + stateError: '#f87171', + end: '#ef4444', +} as const + +function hpCallStateColor(state: CallState): string { + switch (state) { + case 'listening': return HP_CALL.stateListening + case 'thinking': return HP_CALL.stateThinking + case 'speaking': return HP_CALL.stateSpeaking + case 'dialing': + case 'connecting': return HP_CALL.accent + case 'muted': + case 'ended': return HP_CALL.text3 + default: return HP_CALL.text2 + } +} + +// Map the voice-controller's internal machine onto our UI machine. +// Returns null for states we don't want to reflect (OFF — overlay is +// open so "off" is not a meaningful UI frame; render last mapped). +function mapVoiceState(s: VoiceState): CallState | null { + switch (s) { + case 'LISTENING': return 'listening' + case 'THINKING': return 'thinking' + case 'SPEAKING': return 'speaking' + case 'IDLE': return 'listening' + case 'OFF': return null + default: return null + } +} + +// Map the overlay's richer CallState onto the primitive Waveform's +// four-valued mode enum. The primitive doesn't know about dialing / +// connecting / thinking — those all fold to 'idle' (low ambient +// amplitude, faint colour); only the three audible states drive +// distinct waveform modes. +function waveformModeFromCallState(state: CallState): WaveformMode { + if (state === 'listening' || state === 'thinking') return 'listening' + if (state === 'speaking') return 'speaking' + if (state === 'muted') return 'muted' + return 'idle' +} + +function hpCallStateLabel(state: CallState, personaName: string): string { + switch (state) { + case 'dialing': return `calling ${personaName.toLowerCase()}…` + case 'connecting': return 'connecting…' + case 'listening': return 'listening' + case 'thinking': return 'thinking' + case 'speaking': return 'speaking' + case 'muted': return 'microphone off' + case 'ended': return 'call ended' + default: return '' + } +} + +// ── Icons (ported from phone/icons.jsx) ─────────────────────────── + +type IconProps = { size?: number; color?: string } +const IconBase: React.FC = ({ + size = 20, color = 'currentColor', strokeWidth = 1.75, children, +}) => ( + + {children} + +) +const IconPhoneEnd: React.FC = (p) => ( + + + +) +const IconMic: React.FC = (p) => ( + + + + +) +const IconMicOff: React.FC = (p) => ( + + + + + +) +const IconChat: React.FC = (p) => ( + + + +) +const IconBack: React.FC = (p) => ( + + + +) + +// ── ControlBtn (ported from phone/controls.jsx) ─────────────────── + +type Tone = 'neutral' | 'danger' | 'start' | 'accent' +const ControlBtn: React.FC<{ + size?: number + tone?: Tone + active?: boolean + disabled?: boolean + label?: string + ariaLabel?: string + onClick?: () => void + children: React.ReactNode +}> = ({ + size = 48, tone = 'neutral', active = false, disabled, label, ariaLabel, onClick, children, +}) => { + const bg = + tone === 'danger' ? HP_CALL.end : + tone === 'start' ? HP_CALL.stateSpeaking : + tone === 'accent' ? HP_CALL.accent : + active ? 'rgba(255,255,255,0.92)' : HP_CALL.surface2 + const fg = + tone === 'danger' || tone === 'start' ? '#ffffff' : + tone === 'accent' ? '#052c33' : + active ? '#0b0b0c' : HP_CALL.text + const glow = + tone === 'danger' ? 'rgba(239, 68, 68, 0.45)' : + tone === 'start' ? 'rgba(16, 185, 129, 0.45)' : + tone === 'accent' ? 'rgba(34, 211, 238, 0.4)' : + 'rgba(0, 0, 0, 0.3)' + const border = tone === 'neutral' && !active ? `1px solid ${HP_CALL.border}` : 'none' + + return ( +
+ + {label ? ( + {label} + ) : null} +
+ ) +} + +// ── CallAvatar (ported from phone/avatar.jsx) ───────────────────── + +const CallAvatar: React.FC<{ + size?: number + state: CallState + imageUrl?: string | null + accentColor?: string | null +}> = ({ size = 156, state, imageUrl = null, accentColor = null }) => { + const stateColor = hpCallStateColor(state) + const breathes = + state === 'listening' || + state === 'connecting' || + state === 'speaking' || + state === 'dialing' + const showPulseRings = state === 'dialing' + const haloOuter = size + 28 + // seed + hue derivation now live inside the Aura primitive + // (hashHue + moodOffset). This component only owns the halo + + // ring decorations that depend on CallState. + + return ( +
+ {/* Expanding pulse rings — only while dialing. Three rings on + staggered delays so the modal reads as "ringing", not just + "spinning up". */} + {showPulseRings && [0, 1, 2].map(i => ( +