diff --git a/.env.example b/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..946e51b6dbcba58c2bc89c6f17f76a57bc13ca67 --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# Optional realtime input transcription language. +# Defaults to "en". Set to a backend-supported code such as "zh" for Chinese. +# REALTIME_TRANSCRIPTION_LANGUAGE="en" + +PEACHY_REALTIME_PROVIDER=openai +OPENAI_API_KEY= +OPENAI_REALTIME_MODEL=gpt-realtime + +# Hugging Face connection mode: "deployed" or "local". +# Deployed mode uses the built-in Hugging Face server. +# Defaults to "deployed" when unset. +# HF_REALTIME_CONNECTION_MODE="deployed" + +# Direct Hugging Face realtime endpoint for local/LAN backends. +# Accepts either the base URL (`ws://127.0.0.1:8765/v1`) or the full realtime URL +# (`ws://127.0.0.1:8765/v1/realtime`). Used when HF_REALTIME_CONNECTION_MODE=local. +# HF_REALTIME_WS_URL="ws://127.0.0.1:8765/v1/realtime" + +# Hugging Face token for accessing datasets/models +HF_TOKEN= + +# Optional built-in profile selector. Use a folder name from profiles/. +# If unset, the app uses the default profile unless startup settings were saved from the UI. +# REACHY_MINI_CUSTOM_PROFILE=mars_rover + +# Optional external profile/tool directories +# REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY=external_content/external_profiles +# REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY=external_content/external_tools + +# Optional: discover and auto-load all tools found in REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY, +# even if they are not listed in the selected profile's tools.txt. +# This is convenient for downloaded tools used with built-in/default profiles. +# AUTOLOAD_EXTERNAL_TOOLS=1 + +# Minutes of inactivity before Reachy goes to sleep and the app stops. Default 1440 (one day), 0 disables. +# REACHY_MINI_APP_TIMEOUT_MINUTES=1440 diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..48e90d16af2aec310cd69aa4ebee2d26c2b98fb7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +docs/assets/reachy_mini_dance.gif filter=lfs diff=lfs merge=lfs -text diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..ecb85049ea169e08a71537170536505aaebd81b5 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,10 @@ +include LICENSE +include README.md +include index.html +include style.css +include src/peachy/.env.example +recursive-include src/peachy/static * +recursive-include src/peachy/profiles * +recursive-include docs/assets * +include tests/_package_build.py +global-exclude *.py[cod] diff --git a/README.md b/README.md index 870a4882f5f8362a8b150bc02e2157557cae2bad..d0fe35035a1f8ec46d7f69be17ed4827f3fa4443 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,411 @@ --- title: Peachy -emoji: 📈 -colorFrom: green -colorTo: red +emoji: 🍑 +colorFrom: yellow +colorTo: pink sdk: static pinned: false +short_description: Peachy for Reachy Mini +suggested_storage: large +tags: + - reachy_mini + - reachy_mini_python_app --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# Peachy + +Peachy is an on-robot conversational assistant for the Reachy Mini robot, combining realtime voice backends and choreographed motion libraries. + +![Reachy Mini Dance](docs/assets/reachy_mini_dance.gif) + +## Table of contents +- [Overview](#overview) +- [Architecture](#architecture) +- [Install and configure](#install-and-configure) +- [Installation](#installation) +- [Configuration](#configuration) +- [Running the app](#running-the-app) +- [LLM tools](#llm-tools-exposed-to-the-assistant) +- [Advanced features](#advanced-features) +- [Contributing](#contributing) +- [License](#license) + +## Overview +- Real-time audio conversation loop for low-latency streaming, powered by **OpenAI Realtime** by default with Pollen's Hugging Face backend preserved as an explicit alternative. +- Vision is handled by the realtime backend when the `camera` tool is used. +- Layered motion system queues primary moves (dances, emotions, goto poses, breathing) while blending speech-reactive wobble. +- Async tool dispatch integrates robot motion and camera capture. An optional web UI (`--ui`) provides personality selection, mic control, and settings. + +## Architecture + +The app follows a layered architecture connecting the user, AI services, and robot hardware: + +

+ Architecture Diagram +

+ +## Install and configure + +The Reachy Mini app manager installs the built wheel into its shared apps +environment. Peachy declares `reachy-mini==1.9.0` and does not install the +MuJoCo extra. + +Set configuration through the app settings page, or create the owner-only +state file at `${PEACHY_STATE_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/peachy}/.env`: + +```dotenv +PEACHY_REALTIME_PROVIDER=openai +OPENAI_API_KEY=replace-through-local-settings +OPENAI_REALTIME_MODEL=gpt-realtime +``` + +Use `PEACHY_REALTIME_PROVIDER=huggingface` only to select the preserved Pollen +backend explicitly. Peachy never switches providers automatically. + +## Installation + +> [!IMPORTANT] +> Before using this app, you need to install [Reachy Mini's SDK](https://github.com/pollen-robotics/reachy_mini/).
+> Windows support is currently experimental and has not been extensively tested. Use with caution. + +
+Using uv (recommended) + +Set up the project quickly using [uv](https://docs.astral.sh/uv/): + +```bash +# macOS (Homebrew) +uv venv --python /opt/homebrew/bin/python3.12 .venv + +# Linux / Windows (Python in PATH) +uv venv --python python3.12 .venv + +source .venv/bin/activate +uv sync +``` + +> **Note:** To reproduce the exact dependency set from this repo's `uv.lock`, run `uv sync --frozen`. This ensures `uv` installs directly from the lockfile without re-resolving or updating any versions. + +Include dev dependencies: +```bash +uv sync --group dev +``` + +
+ +
+Using pip + +```bash +python -m venv .venv +source .venv/bin/activate +pip install -e . +``` + +**Install dev dependencies with uv (from `onrobot`):** +```bash +uv sync --group dev +``` + +
+ +### Dependency groups + +| Group | Purpose | Notes | +|-------|---------|-------| +| `dev` | Developer tooling (`pytest`, `ruff`, `mypy`) | Development-only dependencies. Use `uv sync --group dev` from `onrobot`. | + +## Configuration + +The default setup uses OpenAI Realtime and requires `OPENAI_API_KEY`. Save it +through the app settings page or the owner-only state file described above. +Select the Hugging Face backend explicitly before using its connection settings. + +| Variable | Description | +|----------|-------------| +| `PEACHY_REALTIME_PROVIDER` | Realtime backend selector. Defaults to `openai`; set `huggingface` explicitly for the preserved Pollen backend. Peachy does not fail over automatically. | +| `OPENAI_API_KEY` | Required when the selected provider is `openai`. Store it through local settings or the owner-only state `.env`. | +| `OPENAI_REALTIME_MODEL` | OpenAI Realtime model. Defaults to `gpt-realtime`. | +| `REALTIME_TRANSCRIPTION_LANGUAGE` | Optional input transcription language for the realtime backend. Defaults to `en`; set to a backend-supported code such as `zh` for Chinese. | +| `HF_REALTIME_CONNECTION_MODE` | Hugging Face connection selector: `deployed` uses the built-in Hugging Face server; `local` uses `HF_REALTIME_WS_URL`. Defaults to `deployed`. | +| `HF_REALTIME_WS_URL` | Direct websocket endpoint for your own Hugging Face backend. Accepts either a base URL like `ws://127.0.0.1:8765/v1` or the full websocket URL `ws://127.0.0.1:8765/v1/realtime`. Used when `HF_REALTIME_CONNECTION_MODE=local`. | +| `HF_TOKEN` | Optional token for Hugging Face access (for gated/private assets). | +| `REACHY_MINI_APP_TIMEOUT_MINUTES` | Minutes of inactivity before Reachy goes to sleep and the app stops. Defaults to `1440` (one day); set to `0` to disable. | + +### Hugging Face Connection Modes + +After selecting the Hugging Face provider, use the built-in Hugging Face server through the app-managed Space proxy. This is the default Hugging Face connection mode; set it explicitly only when you want to switch back from a saved local endpoint: + +```env +HF_REALTIME_CONNECTION_MODE=deployed +``` + +Run your own realtime voice backend using [speech-to-speech](https://github.com/huggingface/speech-to-speech) on the same machine as the conversation app: + +```env +HF_REALTIME_CONNECTION_MODE=local +HF_REALTIME_WS_URL=ws://127.0.0.1:8765/v1/realtime +``` + +Run your own Hugging Face backend on your laptop and connect to it from Reachy Mini Wireless over the same Wi-Fi network: + +```env +HF_REALTIME_CONNECTION_MODE=local +HF_REALTIME_WS_URL=ws://:8765/v1/realtime +``` + +For that LAN setup, make sure the backend listens on an address reachable from the robot, not only on `127.0.0.1`. + +If the backend stays bound to loopback on your laptop, you can forward it into the robot over SSH instead: + +```bash +ssh -N -R 8765:127.0.0.1:8765 @ +``` + +Then set this on the robot: + +```env +HF_REALTIME_CONNECTION_MODE=local +HF_REALTIME_WS_URL=ws://127.0.0.1:8765/v1/realtime +``` + +In the web UI's Settings view, the Connection section lets you choose either the built-in server or a local `host:port` target. The UI writes `HF_REALTIME_CONNECTION_MODE` for you, and the local path writes `HF_REALTIME_WS_URL` with a default of `localhost:8765`. + +## Running the app + +Activate your virtual environment, then launch: + +```bash +peachy +``` + +> [!TIP] +> Make sure the Reachy Mini daemon is running before launching the app. If you see a `TimeoutError`, it means the daemon isn't started. See [Reachy Mini's SDK](https://github.com/pollen-robotics/reachy_mini/) for setup instructions. + +The app runs in console mode by default. Add `--ui` to also serve a web UI at http://127.0.0.1:7860/ for picking a personality, controlling the mic, and changing settings. All options are described in the CLI table below. + +### CLI options + +| Option | Default | Description | +|--------|---------|-------------| +| `--no-camera` | `False` | Run without camera capture. | +| `--ui` | `False` | Serve the web UI at http://127.0.0.1:7860/, in addition to console mode. | +| `--robot-name` | `None` | Optional. Connect to a specific robot by name when running multiple daemons on the same subnet. See [Multiple robots on the same subnet](#advanced-features). | +| `--debug` | `False` | Enable verbose logging for troubleshooting. | + +### Examples + +```bash +# Audio-only conversation (no camera) +peachy --no-camera + +# Launch with the minimal web UI for personality/mic/settings control +peachy --ui +``` + +## LLM tools exposed to the assistant + +The default profile exposes these tools. Custom profiles can enable a different set in their own `tools.txt`. + +| Tool | Action | Dependencies | +|------|--------|--------------| +| `dance` | Queue a dance from `reachy_mini_dances_library`. | Core install only. | +| `stop_dance` | Clear queued dances. | Core install only. | +| `play_emotion` | Play a recorded emotion clip via Hugging Face datasets. | Core install only. Uses the default open emotions dataset: [`pollen-robotics/reachy-mini-emotions-library`](https://huggingface.co/datasets/pollen-robotics/reachy-mini-emotions-library). | +| `stop_emotion` | Clear queued emotions. | Core install only. | +| `camera` | Capture the latest camera frame and analyze it with the selected realtime backend. | Core install only. Requires the camera (disable with `--no-camera`). | +| `idle_do_nothing` | Explicitly remain idle during an idle turn. Not intended for normal conversation turns. | Core install only. | +| `move_head` | Queue a head pose change (left/right/up/down/front). | Core install only. | +| `head_tracking` | Follow the user's face with the head, or stop following. | Core install only. Requires a daemon with the `vision` extra and a camera. | +| `go_to_sleep` | Run Reachy's sleep movement and stop the current app after an explicit user request. | Core install only. | +| `sweep_look` | Sweep Reachy's head left, right, and back to center. | Bundled default profile tool. | +| `remember` | Save one short, stable fact about the user for future sessions. | Core install only. Stored in the app instance data directory. | +| `forget` | Remove a saved memory fact by matching a short query. | Core install only. | +| `pollen_robotics_reachy_mini_search_tool__search_web` | Search the web and return a short list of results. | Preinstalled MCP Space: `pollen-robotics/reachy-mini-search-tool`. | +| `pollen_robotics_reachy_mini_weather_tool__get_weather` | Report today's weather for a place: current conditions, high and low temperature, and rain chance. | Preinstalled MCP Space: `pollen-robotics/reachy-mini-weather-tool`. | +| `pollen_robotics_reachy_mini_time_tool__get_time` | Report the current time for a timezone or the user's local time, or the difference between two timezones. | Preinstalled MCP Space: `pollen-robotics/reachy-mini-time-tool`. | + +> [!NOTE] +> `remember`/`forget` facts are stored in `memory.v1.json` inside the app's instance data directory (`~/.local/share/peachy/` by default, or the instance path used by the desktop launcher). `forget` only removes facts matched by query. To reset all remembered facts, delete this file. + +## Advanced features + +Built-in motion content is published as open Hugging Face datasets: +- Emotions: [`pollen-robotics/reachy-mini-emotions-library`](https://huggingface.co/datasets/pollen-robotics/reachy-mini-emotions-library) +- Dances: [`pollen-robotics/reachy-mini-dances-library`](https://huggingface.co/datasets/pollen-robotics/reachy-mini-dances-library) + +
+Custom profiles + +Create custom profiles with dedicated instructions and enabled tools. + +For normal usage, select a profile from the UI and save it for startup. That selection is persisted in `startup_settings.json`. + +If no startup settings have been saved yet, `REACHY_MINI_CUSTOM_PROFILE=` selects a named built-in package profile unless an external profiles directory is configured. If neither is set, Peachy uses its built-in default profile. + +Each profile should include `instructions.txt` (prompt text). If that file is missing or empty, the app logs a warning and falls back to Peachy's package-local default profile. `greeting.txt` is optional and controls how the robot should start the conversation after the backend connects. `tools.txt` (list of allowed tools) is recommended. If missing for a non-default profile, the app falls back to the built-in package-local default tools. Profiles can optionally contain custom tool implementations. + +**Startup greeting:** + +On startup, once the realtime backend is connected and ready, the app sends the active profile's `greeting.txt` as an internal text turn so the model opens with a fresh spoken greeting. Keep this file as a short instruction, not a fixed script, for example: +``` +Greet me warmly in one sentence, in character, and vary the wording each time. +``` +If `greeting.txt` is missing, the app uses the built-in default greeting prompt. + +**Enabling tools:** + +List enabled tools in `tools.txt`, one per line. Prefix with `#` to comment out: +``` +play_emotion +# move_head + +# My custom tool defined locally +sweep_look +``` +Tools are resolved first from Python files in the profile folder (custom tools), then from the core library `src/peachy/tools/` (like `dance`, `camera`). +Installed Hugging Face Space tools can also be enabled here after you add them with `tool-spaces`. + +**Custom tools:** + +On top of built-in tools found in the core library, you can implement custom tools specific to your profile by adding Python files in the profile folder. +Custom tools must subclass `peachy.tools.core_tools.Tool` (see that module for the interface). + +**Edit personalities from the UI:** + +When running with `--ui`, the Home view lists package-local built-ins plus instance-directory user personalities: +- Tap a card to apply that personality and start talking. +- Tap "Custom" to create a new personality by entering a name, instructions, and an optional startup greeting prompt. It copies `tools.txt` from the `default` profile and stores the files under `user_personalities//` in the app instance directory (next to `.env`/`startup_settings.json`). + +In a source checkout, built-ins live under `src/peachy/profiles/`; installed built-ins are read-only package data. Author durable profiles through the UI or an external profiles directory. + +Note: switching a personality reloads its instructions and tools in place via a quick backend reconnect — no app restart. Editing the active profile's files on disk needs a re-select (or restart) to apply. + +
+ +
+Locked profile mode + +To create a locked variant of the app that cannot switch profiles, edit `src/peachy/config.py` and set the `LOCKED_PROFILE` constant to the desired profile name: +```python +LOCKED_PROFILE: str | None = "mars_rover" # Lock to this profile +``` +When `LOCKED_PROFILE` is set, the app always uses that profile, ignoring saved startup settings, `REACHY_MINI_CUSTOM_PROFILE`, and the web UI. The UI shows "(locked)" and disables all profile editing controls. +This is useful for creating dedicated clones of the app with a fixed personality. Clone scripts can simply edit this constant to lock the variant. + +
+ +
+External profiles and tools + +You can extend the app with profiles/tools stored outside the repository defaults. Source checkout built-ins are under `src/peachy/profiles/`; installed built-ins are read-only package data. Author durable profiles through the UI or an external profiles directory. + +- Core tools are under `src/peachy/tools/`. + +**Recommended layout:** + +```text +external_content/ +├── external_profiles/ +│ └── my_profile/ +│ ├── instructions.txt +│ ├── greeting.txt # optional startup greeting prompt +│ ├── tools.txt # optional (see fallback behavior below) +│ └── voice.txt # optional +├── external_tools/ +│ └── my_custom_tool.py +└── installed_tool_spaces.json +``` + +**Environment variables:** + +Set these values in your `.env` when you want env-driven external profile/tool selection: + +```env +# Optional fallback/manual profile selector: +REACHY_MINI_CUSTOM_PROFILE=my_profile +REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY=./external_content/external_profiles +REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY=./external_content/external_tools +# Optional convenience mode: +# AUTOLOAD_EXTERNAL_TOOLS=1 +``` + +**Loading behavior:** + +- **Default/strict mode**: `tools.txt` defines enabled tools explicitly. Every name in `tools.txt` must resolve to either a built-in tool (`src/peachy/tools/`) or an external tool module in `REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY`. +- **Convenience mode** (`AUTOLOAD_EXTERNAL_TOOLS=1`): all valid `*.py` tool files in `REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY` are auto-added. +- **External profile fallback**: if the selected external profile has no `tools.txt`, the app falls back to the built-in package-local default tools. +- **Duplicate safety**: every loaded tool class must expose a unique `Tool.name`. The app now fails fast if two tool implementations claim the same tool name. + +This supports both: +1. Local external tools used with built-in/default profile. +2. Local external profiles used with built-in default tools. + +
+ +
+Hugging Face Space tools + +You can install MCP-compatible Hugging Face Spaces as remote tool sources for this app. Private Spaces work too, as long as `HF_TOKEN` is set (or you have run `hf auth login`) for an account that can access them. + +```bash +# install + enable in active profile +peachy tool-spaces add + +# enable in a specific profile +peachy tool-spaces add --profile NAME + +# install without enabling +peachy tool-spaces add --install-only + +# list installed spaces +peachy tool-spaces list + +# remove an installed space +peachy tool-spaces remove owner/space-name +``` + +The bundled Pollen Spaces are enabled by default and resolve from static specs, so startup needs no Hugging Face discovery. For custom Spaces, the app validates the slug through the Hugging Face Hub, probes the standard MCP endpoint (sending the HF token only to private Spaces), discovers tools, enables them in the active profile's `tools.txt`, and writes the installed Space to: + +- `installed_tool_spaces.json` in the managed app instance directory +- `external_content/installed_tool_spaces.json` in terminal mode + +Recommended tags for discoverability on Hugging Face: + +- `reachy-mini-tool` +- `mcp` + +These tags are advisory only. Installation still relies on successful MCP validation, not on tag presence. + +> [!NOTE] +> Preinstalled Pollen Spaces can be removed like any other (`tool-spaces remove pollen-robotics/reachy-mini-weather-tool`) or delete `installed_tool_spaces.json` to restore all defaults. + +
+ +
+Multiple robots on the same subnet + +If you run multiple Reachy Mini daemons on the same network, use: + +```bash +peachy --robot-name +``` + +`` must match the daemon's `--robot-name` value so the app connects to the correct robot. + +
+ +## Contributing + +We welcome bug fixes, features, profiles, and documentation improvements. From `onrobot`, set up the development environment and run the local checks: + +```bash +uv sync --group dev +.venv/bin/python -m pytest tests +.venv/bin/ruff check src tests +.venv/bin/mypy +``` + +## License + +Apache 2.0 diff --git a/UPSTREAM.md b/UPSTREAM.md new file mode 100644 index 0000000000000000000000000000000000000000..94319be0f4cf666d59219c475a79ee0bfa6af2c5 --- /dev/null +++ b/UPSTREAM.md @@ -0,0 +1,24 @@ +# Upstream provenance + +Peachy is a modified source fork of [Pollen Robotics' Reachy Mini Conversation App](https://github.com/pollen-robotics/reachy_mini_conversation_app). + +- Upstream tag: `v0.10.0` +- Immutable source commit: `40ea94e49213f77ae83d21a81c1766b2e4f25e49` + +## Copied input mapping + +- `src/reachy_mini_conversation_app/** -> onrobot/src/peachy/**` +- `tests/** -> onrobot/tests/**` +- `profiles/** -> onrobot/src/peachy/profiles/**` +- `.env.example -> onrobot/.env.example` +- `.env.example -> onrobot/src/peachy/.env.example` +- `LICENSE -> onrobot/LICENSE` +- `index.html -> onrobot/index.html` +- `style.css -> onrobot/style.css` +- `docs/assets/conversation_app_arch.svg -> onrobot/docs/assets/conversation_app_arch.svg` +- `docs/assets/reachy_mini_dance.gif -> onrobot/docs/assets/reachy_mini_dance.gif` + +This Apache-2.0-licensed source has been modified by Peachy contributors for namespace, packaging, and provider integration. + +External profile/tool Python files that import `reachy_mini_conversation_app.*` +must use `peachy.*` in this spike. No compatibility namespace is installed. diff --git a/docs/assets/conversation_app_arch.svg b/docs/assets/conversation_app_arch.svg new file mode 100644 index 0000000000000000000000000000000000000000..c36f8cc5ae6f75ce40040c625f5a22e417f25379 --- /dev/null +++ b/docs/assets/conversation_app_arch.svg @@ -0,0 +1,102 @@ +

audio stream

audio stream

audio stream

tool calls

defines enabled tools

movement
requests

image for
analysis

camera
frames

commands

results

User
Person interacting with system

UI Layer
Web UI / Console

Realtime Backend
Audio + Tool Calls + Vision

Motion Control
Audio Sync

Tool Layer
Built-in tools + profile-local tools
+ external tools (optional)

Selected Profile
built-in or external
instructions.txt + tools.txt

reachy_mini
Robot Control Library

\ No newline at end of file diff --git a/docs/assets/reachy_mini_dance.gif b/docs/assets/reachy_mini_dance.gif new file mode 100644 index 0000000000000000000000000000000000000000..d801b57f4de4bcba30589b44522f99975ba7bdca --- /dev/null +++ b/docs/assets/reachy_mini_dance.gif @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75914c3cb7af982e0b1c6369e25fc46d8c08a0ab5ad022240ae9c1a0d93967c3 +size 3930400 diff --git a/docs/legacy-regression-bootstrap.md b/docs/legacy-regression-bootstrap.md new file mode 100644 index 0000000000000000000000000000000000000000..530f1e0103d1ac5245b2df45b9f8930d9b29de61 --- /dev/null +++ b/docs/legacy-regression-bootstrap.md @@ -0,0 +1,91 @@ +# Legacy root regression bootstrap + +The repository-root Mac/simulator app targets the old FastRTC engine and SDK +1.8.0. Its tests must not run in the `onrobot` SDK 1.9.0 environment. + +```bash +PEACHY_LEGACY_ENGINE_DIR="$(mktemp -d /tmp/peachy-engine-v030.XXXXXX)" +PEACHY_LEGACY_VENV_DIR="$(mktemp -d /tmp/peachy-legacy-venv.XXXXXX)" +git clone --depth 1 --branch v0.3.0 \ + https://github.com/pollen-robotics/reachy_mini_conversation_app.git \ + "$PEACHY_LEGACY_ENGINE_DIR" +test "$(git -C "$PEACHY_LEGACY_ENGINE_DIR" rev-parse HEAD)" = \ + "93674f987cd3172296e503d81674ece3ab58b4df" +uv venv --python python3.12 "$PEACHY_LEGACY_VENV_DIR" +uv pip install --python "$PEACHY_LEGACY_VENV_DIR/bin/python" \ + -r requirements.txt -r requirements-dev.txt -e "$PEACHY_LEGACY_ENGINE_DIR" +uv pip install --python "$PEACHY_LEGACY_VENV_DIR/bin/python" \ + --no-deps -e . +test "$("$PEACHY_LEGACY_VENV_DIR/bin/python" -c \ + 'import importlib.metadata as m; print(m.version("reachy-mini-conversation-app"))')" = \ + "0.3.0" +test "$("$PEACHY_LEGACY_VENV_DIR/bin/python" -c \ + 'import importlib.metadata as m; print(m.version("reachy-mini"))')" = \ + "1.8.0" +"$PEACHY_LEGACY_VENV_DIR/bin/python" -m pytest tests -q -m "not e2e" +``` + +Expected: the exact SHA and package-version checks succeed, and this diagnostic +non-daemon baseline passes. It is not a silent substitute for the full default +gate described below. The root editable install uses `--no-deps`, so +the legacy engine remains 0.3.0 and the SDK remains 1.8.0. Keep both temporary +paths until failures are diagnosed; remove them manually only after results have +been recorded. This bootstrap does not install the new `onrobot` package and +does not modify the repository-root application. + +## Full default gate and simulator prerequisite + +The original required gate is unfiltered: + +```bash +REACHY_VENV_PY="$PEACHY_LEGACY_VENV_DIR/bin/python" \ + "$PEACHY_LEGACY_VENV_DIR/bin/python" -m pytest tests -q +``` + +The two default simulator tests use the repository's existing daemon fixture, +which launches +`reachy_mini_env/bin/python` unless `REACHY_VENV_PY` names another provisioned +SDK 1.8.0 simulator environment. The fixture starts: + +```text +python -m reachy_mini.daemon.app.main --sim --headless --no-media +``` + +It treats port 8000 as ready, then requires the SDK client to receive initial +joint-position and head-pose state. Provision `reachy_mini_env` with the root +README workflow and confirm no daemon is already occupying port 8000 before +running the full gate or the focused diagnostic: + +```bash +REACHY_VENV_PY="$PWD/reachy_mini_env/bin/python" \ + "$PEACHY_LEGACY_VENV_DIR/bin/python" -m pytest \ + tests/e2e/test_m5_local_brain.py tests/e2e/test_sim_smoke.py -q -m e2e +``` + +These tests require local socket binding and a working headless MuJoCo daemon. +Other opt-in E2E, model, voice, macOS-app, and robot markers retain the separate +authorization and hardware/model prerequisites documented in the root README. + +### macOS simulator blocker and acceptance decision + +On the reviewed Apple Silicon host, both the Python and `mjpython` launchers +reported that SDK 1.8.0's daemon started successfully and bound port 8000, but a +15-second `localhost_only` SDK probe never received initial joint/head state. +Consequently the unfiltered gate completed with `893 passed, 56 skipped, 2 +errors`; only these setup nodes errored: + +- `tests/e2e/test_m5_local_brain.py::test_local_brain_moves_head_and_emits_audio` +- `tests/e2e/test_sim_smoke.py::test_sim_moves_antennas` + +The root README records the working macOS fallback as a non-headless daemon +started with `mjpython` in a real Terminal. That interactive GUI prerequisite is +outside a headless test worker. To complete the two nodes, provision the +documented worktree-local `reachy_mini_env`, start that GUI daemon in a real +Terminal, wait for actual joint/head state, and then run the focused command +above; the fixture will reuse the daemon already on port 8000. + +Task 9 received explicit human acceptance of this amendment: the passing +`-m "not e2e"` baseline and the unfiltered `893 passed, 56 skipped, 2 errors` +result are accepted with the two simulator setup errors recorded as an external +macOS GUI-daemon prerequisite. The two focused E2E nodes remain required GUI +simulator follow-up; this record does not claim the full default gate passed. diff --git a/docs/openai-realtime-smoke.md b/docs/openai-realtime-smoke.md new file mode 100644 index 0000000000000000000000000000000000000000..1996876851d2293ccb32d705476a48650e1016b2 --- /dev/null +++ b/docs/openai-realtime-smoke.md @@ -0,0 +1,86 @@ +# OpenAI Realtime adapter smoke + +This is a paid, opt-in test. It builds one client and opens at most one WebSocket +session attempt—there is no retry loop. It limits each response to 64 output +tokens, disables the startup greeting and tools, feeds one short checked-in voice +clip, feeds it once more to exercise barge-in, and observes at most two +`response.created` events (maximum 128 output tokens). A test-local wrapper +counts those events directly at the handler boundary, so the bounded console +activity queue cannot lose them; a third event cancels its active response and +fails the smoke immediately. It tears the session down within the test's +45-second outer timeout. The approved ceiling for one run is **$0.10**. + +## TLS trust on macOS Python.org installs + +The guarded smoke sets `SSL_CERT_FILE` to the active `onrobot/.venv` certifi CA +bundle after the spend/key guards pass and before it constructs or starts the +Realtime handler. This is test-local setup: it does not change the production +handler or persist an environment setting after the pytest process exits. + +This setup is necessary on affected Python.org macOS installations where the +configured framework CA path, such as +`/Library/Frameworks/Python.framework/Versions/3.12/etc/openssl/cert.pem`, is +missing. In that condition, Python's default SSL context can fail with +`CERTIFICATE_VERIFY_FAILED: unable to get local issuer certificate` even though +the venv's certifi bundle is valid. + +As a system-level alternative, run the installer matching the Python.org version +once (for Python 3.12, open `/Applications/Python 3.12/Install Certificates.command`). +That repairs the framework interpreter's default certificate link. The manual +smoke still selects its venv certifi bundle explicitly so its trust source is +deterministic. + +## Realtime event ordering + +Response audio can reach the playback sink before the server publishes +`conversation.item.input_audio_transcription.completed`. After first audio, the +smoke therefore waits up to 10 seconds for the event bus reason +`user_transcription_completed` instead of taking an immediate queue snapshot. +It retains every earlier reason; if completion is truly absent, the assertion +reports the missing target and the reasons observed before the deadline. This +bounded wait remains inside the unchanged 45-second outer pytest timeout and +does not weaken the later barge-in or teardown checks. + +Before running, review the [current official Realtime pricing page](https://developers.openai.com/api/docs/pricing). Do not run if +the two short inputs plus at most 128 output tokens could exceed $0.10. Use a +restricted project key, never paste the key into a command or committed file, +and authorize the ceiling explicitly: + +```bash +export OPENAI_API_KEY='set in your private shell or key manager' +export RUN_OPENAI_REALTIME=1 +export PEACHY_LIVE_SPEND_APPROVED=0.10 +onrobot/.venv/bin/python -m pytest -o addopts= -m live_openai \ + onrobot/tests/manual/test_openai_realtime_live.py -v --timeout=45 +``` + +Pass criteria: + +- the checked-in user recording reaches completed transcription; +- a delegating spy around the real console resampler proves 24 kHz provider input + reaches the 48 kHz sink with exactly twice the sample count; +- before the second feed, real response audio is pending both in the fake player + and in the handler output queue; +- second-feed speech start invokes exactly one additional player clear, empties + both pending layers, and that clear count remains exact through the complete + second-feed interruption lifecycle; +- releasing the playback gate during active speech does not replay cancelled + response audio; provider-audio provenance is captured at the event/queue + boundary, so a delayed cancelled frame remains attributed to its original + response even after a later `response.created` and is not mistaken for + legitimate later-response audio; +- a lossless test-local guard observes one or two capped `response.created` + events, mechanically keeping maximum output at 128 tokens; and +- teardown leaves no active connection or tool-manager task and surfaces any + unexpected session/playback task exception (intentional cancellations and + orderly completion are allowed). + +For a listening check, route the sink through a real Reachy Mini only after the +automated adapter gate passes. Previously queued speech must stop and must not +resume after the interruption. + +Known spike boundary: [official WebSocket guidance](https://developers.openai.com/api/docs/guides/realtime-conversations#interruption-and-truncation) also recommends sending +`conversation.item.truncate` with the amount of audio actually played so server +conversation state excludes unheard audio. This spike proves audible cancellation +and late-delta suppression but does not yet maintain a playback clock/item +truncation offset; record that as a required follow-on if the adapter gate passes. diff --git a/index.html b/index.html index b0c4b3666032a737f3903db53e6a8a9272483e28..935ba228f77f67b4ae7b1cb1ecd838d75dcde979 100644 --- a/index.html +++ b/index.html @@ -1,19 +1,125 @@ - - - - My static Space - - - -
-

Welcome to your static Space!

-

You can modify this app directly by editing index.html in the Files and versions tab.

-

- Also don't forget to check the - Spaces documentation. + + + + + Reachy Mini Conversation App + + + + + + + +

+
+
+ 🤖 + Reachy Mini +
+
Realtime voice · Vision aware · Expressive motion
+
+
+
+

Conversation App

+

Talk, see, and move together.

+

+ A friendly, camera-aware companion for Reachy Mini. Chat out loud, watch it follow faces, dance, or react with recorded emotions—all while streaming transcripts in a clean web UI. +

+
+ Explore features + See how it feels +
+
+ Low-latency voice loop + Camera insights on demand + Choreographed dances & emotions + Personality profiles via web UI +
+
+
+
+ Reachy Mini dancing +

Reachy Mini can move, dance, and emote while holding a natural conversation.

+
+
+
+
+ +
+
+

What’s inside

+

All-in-one conversational layer for your robot

+

+ The app blends realtime speech, vision, and motion so Reachy Mini feels present..

- +
+
+ 🎤 +

Natural voice chat

+

Talk freely and get fast, high-quality replies powered by realtime models.

+
+
+ 🎥 +

Vision-aware replies

+

Ask the camera tool to see what’s in front, track a face, or keep attention on whoever is speaking.

+
+
+ 💃 +

Expressive motion

+

Queue dances, play recorded emotions while Reachy listens and talks.

+
+
+ 🧠 +

Personalities on demand

+

Switch conversation styles through profiles and decide which tools (dance, camera, tracking) each persona can use.

+
+
+ 🌐 +

Ready for your setup

+

Works with wired or wireless Reachy Mini, and can run vision locally or through the default cloud model.

+
+
+
+ +
+
+
+

How it feels

+

From hello to helpful in seconds

+
    +
  • 👋 Say “Hey Reachy” and start chatting—no extra setup in the moment.
  • +
  • 👀 Ask what it sees; it can peek through the camera or keep focus on your face.
  • +
  • 🎭 Trigger emotions or dance breaks to keep the conversation lively.
  • +
  • 📝 Follow along with live transcripts in the web UI or run audio-only from the console.
  • +
+
+
+

Where it shines

+

Great for demos, teaching, and playful exploration

+

+ Show off how Reachy Mini listens, responds, and moves in sync. Whether you’re guiding a class, hosting a booth, or experimenting at home, the app keeps the robot expressive without juggling scripts or joystick controls. +

+
+ Live conversation + Face tracking + Camera tool + Dance library + Profiles & tools +
+
+
+
+ +
+

+ Reachy Mini Conversation App by Pollen Robotics. + Explore more apps on Hugging Face Spaces. +

+
+ + + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..ad073540899f3f2a079815dc7bd9edfd525934c9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,79 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "peachy" +version = "0.1.0" +description = "Peachy — on-robot conversational assistant for Reachy Mini" +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.12" +dependencies = [ + "huggingface-hub>=1.17.0", + "httpx>=0.28.1", + "python-dotenv", + "openai==2.28.0", + "reachy_mini_dances_library", + "reachy_mini_toolbox", + "reachy-mini==1.9.0", + "mcp>=1.27.1", + "numpy", + "scipy", +] +keywords = ["reachy-mini-app"] + +[project.scripts] +peachy = "peachy.main:main" + +[project.entry-points."reachy_mini_apps"] +peachy = "peachy.main:Peachy" + +[dependency-groups] +dev = [ + "build", + "certifi", + "pytest", + "pytest-asyncio", + "pytest-timeout", + "ruff==0.12.0", + "mypy==1.18.2", +] + +[tool.setuptools] +package-dir = { "" = "src" } +include-package-data = false + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +peachy = [ + "static/*", + "static/**/*", + ".env.example", + "profiles/**/*.txt", +] + +[tool.setuptools.exclude-package-data] +"*" = ["**/__pycache__/*", "**/*.pyc"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" +addopts = "-m 'not live_openai'" +markers = [ + "live_openai: opt-in paid OpenAI Realtime smoke; never CI", +] + +[tool.ruff] +line-length = 119 +exclude = [".venv", "dist", "build", "**/__pycache__", "*.egg-info", ".mypy_cache", ".pytest_cache"] + +[tool.mypy] +python_version = "3.12" +files = ["src/"] +ignore_missing_imports = true +strict = true +show_error_codes = true +warn_unused_ignores = true diff --git a/src/peachy/.env.example b/src/peachy/.env.example new file mode 100644 index 0000000000000000000000000000000000000000..946e51b6dbcba58c2bc89c6f17f76a57bc13ca67 --- /dev/null +++ b/src/peachy/.env.example @@ -0,0 +1,36 @@ +# Optional realtime input transcription language. +# Defaults to "en". Set to a backend-supported code such as "zh" for Chinese. +# REALTIME_TRANSCRIPTION_LANGUAGE="en" + +PEACHY_REALTIME_PROVIDER=openai +OPENAI_API_KEY= +OPENAI_REALTIME_MODEL=gpt-realtime + +# Hugging Face connection mode: "deployed" or "local". +# Deployed mode uses the built-in Hugging Face server. +# Defaults to "deployed" when unset. +# HF_REALTIME_CONNECTION_MODE="deployed" + +# Direct Hugging Face realtime endpoint for local/LAN backends. +# Accepts either the base URL (`ws://127.0.0.1:8765/v1`) or the full realtime URL +# (`ws://127.0.0.1:8765/v1/realtime`). Used when HF_REALTIME_CONNECTION_MODE=local. +# HF_REALTIME_WS_URL="ws://127.0.0.1:8765/v1/realtime" + +# Hugging Face token for accessing datasets/models +HF_TOKEN= + +# Optional built-in profile selector. Use a folder name from profiles/. +# If unset, the app uses the default profile unless startup settings were saved from the UI. +# REACHY_MINI_CUSTOM_PROFILE=mars_rover + +# Optional external profile/tool directories +# REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY=external_content/external_profiles +# REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY=external_content/external_tools + +# Optional: discover and auto-load all tools found in REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY, +# even if they are not listed in the selected profile's tools.txt. +# This is convenient for downloaded tools used with built-in/default profiles. +# AUTOLOAD_EXTERNAL_TOOLS=1 + +# Minutes of inactivity before Reachy goes to sleep and the app stops. Default 1440 (one day), 0 disables. +# REACHY_MINI_APP_TIMEOUT_MINUTES=1440 diff --git a/src/peachy/__init__.py b/src/peachy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7e334df1df4868f72b8c26b7e2d9f671f4add762 --- /dev/null +++ b/src/peachy/__init__.py @@ -0,0 +1,2 @@ +"""Nothing (for ruff).""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. diff --git a/src/peachy/app_lifecycle.py b/src/peachy/app_lifecycle.py new file mode 100644 index 0000000000000000000000000000000000000000..f6a6ad767031f1b730512941e2ceb9c1e79da077 --- /dev/null +++ b/src/peachy/app_lifecycle.py @@ -0,0 +1,85 @@ +"""Helpers for app startup and shutdown lifecycle behavior.""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +import asyncio +import logging +import urllib.error +import urllib.request + +import numpy as np +import numpy.typing as npt + +from reachy_mini import ReachyMini +from reachy_mini.reachy_mini import SLEEP_HEAD_POSE +from reachy_mini.utils.interpolation import distance_between_poses +from peachy.tools.core_tools import ToolDependencies +from peachy.tools.go_to_sleep import GoToSleep + + +_STOP_CURRENT_APP_PATH = "/api/apps/stop-current-app" +_STOP_CURRENT_APP_TIMEOUT_S = 2.0 +_SLEEP_HEAD_TRANSLATION_TOLERANCE_M = 0.05 +_SLEEP_HEAD_ROTATION_TOLERANCE_RAD = 0.35 + + +def request_stop_current_app(robot: ReachyMini, logger: logging.Logger) -> bool: + """Request the Reachy Mini daemon to stop the current app.""" + stop_current_app_url = f"http://{robot.client.host}:{robot.client.port}{_STOP_CURRENT_APP_PATH}" + request = urllib.request.Request(stop_current_app_url, method="POST") + try: + with urllib.request.urlopen(request, timeout=_STOP_CURRENT_APP_TIMEOUT_S) as response: + response.read() + except urllib.error.URLError as e: + logger.error("Failed to request current app stop via %s: %s", stop_current_app_url, e) + return False + + logger.info("Requested current app stop via %s", stop_current_app_url) + return True + + +def _is_sleep_head_pose(head_pose: npt.ArrayLike) -> bool: + try: + current_head_pose: npt.NDArray[np.float64] = np.asarray(head_pose, dtype=np.float64) + except (TypeError, ValueError): + return False + + if current_head_pose.shape != (4, 4): + return False + + pose_distances = distance_between_poses(current_head_pose, SLEEP_HEAD_POSE) + translation_distance = float(pose_distances[0]) + rotation_angle = float(pose_distances[1]) + return ( + translation_distance <= _SLEEP_HEAD_TRANSLATION_TOLERANCE_M + and rotation_angle <= _SLEEP_HEAD_ROTATION_TOLERANCE_RAD + ) + + +def wake_up_if_sleeping(robot: ReachyMini, logger: logging.Logger) -> bool: + """Run the SDK wake-up movement when Reachy starts from the sleep pose.""" + try: + head_pose = robot.get_current_head_pose() + except Exception as e: + logger.warning("Could not read robot pose before startup wake-up check: %s", e) + return False + + if not _is_sleep_head_pose(head_pose): + return False + + logger.info("Robot is in sleep pose; running wake-up movement.") + try: + robot.wake_up() + except Exception as e: + logger.error("Failed to run wake-up movement: %s", e) + return False + return True + + +def run_go_to_sleep_tool(deps: ToolDependencies, logger: logging.Logger) -> dict[str, object]: + """Run the shared go_to_sleep tool from synchronous shutdown paths.""" + try: + return asyncio.run(GoToSleep()(deps)) + except Exception as e: + logger.error("Failed to run go_to_sleep tool during shutdown: %s", e) + return {"error": f"go_to_sleep failed: {type(e).__name__}: {e}"} diff --git a/src/peachy/audio/__init__.py b/src/peachy/audio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..215891716d270d9e3b7d27463ca296ec7cc1a7e7 --- /dev/null +++ b/src/peachy/audio/__init__.py @@ -0,0 +1 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. diff --git a/src/peachy/audio/resampling.py b/src/peachy/audio/resampling.py new file mode 100644 index 0000000000000000000000000000000000000000..b8e8a472055e064af70a189374d88784e2c79ad8 --- /dev/null +++ b/src/peachy/audio/resampling.py @@ -0,0 +1,55 @@ +"""Provider-neutral mono normalization and audio sample-rate conversion.""" + +from __future__ import annotations + +from typing import cast + +import numpy as np +from numpy.typing import NDArray +from scipy.signal import resample + +from peachy.streaming import AudioArray, audio_to_float32 + + +def audio_to_mono(audio: AudioArray) -> AudioArray: + """Return a contiguous mono array while preserving dtype.""" + if audio.ndim == 1: + return cast(AudioArray, np.ascontiguousarray(audio)) + if audio.ndim != 2: + raise ValueError("Audio must be one- or two-dimensional") + if audio.size == 0: + return cast(AudioArray, np.empty(0, dtype=audio.dtype)) + channels_last = audio.T if audio.shape[1] > audio.shape[0] else audio + mono = channels_last[:, 0] + return cast(AudioArray, np.ascontiguousarray(mono)) + + +def resample_audio( + audio: AudioArray, + source_rate: int, + target_rate: int, +) -> NDArray[np.float32]: + """Return contiguous mono float32 audio at `target_rate`.""" + if source_rate <= 0 or target_rate <= 0: + raise ValueError("Audio sample rates must be positive") + mono = audio_to_float32(audio_to_mono(audio)) + if mono.size == 0: + return np.ascontiguousarray(mono, dtype=np.float32) + if source_rate == target_rate: + return np.ascontiguousarray(mono, dtype=np.float32) + target_samples = round(mono.shape[0] * target_rate / source_rate) + if target_samples == 0: + return np.empty(0, dtype=np.float32) + converted = resample(mono, target_samples) + return np.ascontiguousarray(converted, dtype=np.float32) + + +def pcm16_at_rate( + audio: AudioArray, + source_rate: int, + target_rate: int, +) -> NDArray[np.int16]: + """Return clipped, contiguous mono signed PCM16 at `target_rate`.""" + converted = resample_audio(audio, source_rate, target_rate) + clipped = np.clip(converted, -1.0, 1.0) + return np.ascontiguousarray(clipped * 32767.0, dtype=np.int16) diff --git a/src/peachy/audio/startup_config.py b/src/peachy/audio/startup_config.py new file mode 100644 index 0000000000000000000000000000000000000000..a656cd80ab4636d2e77640bd429465a9b88d19c6 --- /dev/null +++ b/src/peachy/audio/startup_config.py @@ -0,0 +1,66 @@ +"""Startup configuration for the Reachy Mini audio processor.""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import logging +from collections.abc import Sequence + + +AudioControlValue = float | int +AudioStartupParameter = tuple[str, tuple[AudioControlValue, ...]] +WRITE_SETTLE_SECONDS = 0.1 + +AUDIO_STARTUP_CONFIG: tuple[AudioStartupParameter, ...] = ( + ("PP_AGCMAXGAIN", (10.0,)), + ("PP_MIN_NS", (0.8,)), + ("PP_MIN_NN", (0.8,)), + ("PP_GAMMA_E", (0.5,)), + ("PP_GAMMA_ETAIL", (0.5,)), + ("PP_NLATTENONOFF", (0,)), + ("PP_MGSCALE", (4.0, 1.0, 1.0)), +) + + +def apply_audio_startup_config( + robot: object, + *, + logger: logging.Logger | None = None, + verify: bool = True, + write_settle_seconds: float = WRITE_SETTLE_SECONDS, +) -> bool: + """Apply the tuned XVF3800 audio configuration for the conversation app.""" + log = logger or logging.getLogger(__name__) + audio = getattr(getattr(robot, "media", None), "audio", None) + + if audio is None: + log.warning("Skipping Reachy audio startup config: robot media audio is unavailable.") + return False + + apply_audio_config = getattr(audio, "apply_audio_config", None) + if not callable(apply_audio_config): + log.warning("Skipping Reachy audio startup config: SDK audio config API is unavailable.") + return False + + try: + applied = bool( + apply_audio_config( + AUDIO_STARTUP_CONFIG, + verify=verify, + write_settle_seconds=write_settle_seconds, + ) + ) + except Exception as exc: + log.warning("Skipping Reachy audio startup config: SDK audio config failed: %s", exc) + return False + + if applied: + log.info("Applied Reachy audio startup config: %s", _format_config(AUDIO_STARTUP_CONFIG)) + else: + log.warning("Reachy audio startup config was not applied.") + + return applied + + +def _format_config(config: Sequence[AudioStartupParameter]) -> str: + return ", ".join(f"{name}={' '.join(str(value) for value in values)}" for name, values in config) diff --git a/src/peachy/config.py b/src/peachy/config.py new file mode 100644 index 0000000000000000000000000000000000000000..5227a4b9ccaf2dc04cf7a897dac0a6764f4216c1 --- /dev/null +++ b/src/peachy/config.py @@ -0,0 +1,489 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +import os +import sys +import logging +from pathlib import Path +from dataclasses import dataclass +from urllib.parse import urlsplit, parse_qsl, urlunsplit +from importlib.resources import files + +from dotenv import find_dotenv, load_dotenv +from peachy.state import resolve_state_root + + +# Locked profile: set to a profile name (e.g., "astronomer") to lock the app +# to that profile and disable all profile switching. Leave as None for normal behavior. +LOCKED_PROFILE: str | None = None +PROJECT_ROOT = Path(__file__).parents[2].resolve() + + +def _is_source_checkout_root(root: Path) -> bool: + """Return whether the given root looks like this project's source checkout.""" + return (root / "pyproject.toml").is_file() and (root / "src" / "peachy").is_dir() + + +def _packaged_profiles_directory() -> Path | None: + """Return Peachy's wheel-owned built-in profiles directory.""" + try: + return Path(str(files("peachy").joinpath("profiles"))) + except Exception: + return None + + +def _resolve_default_profiles_directory() -> Path: + """Resolve built-in profiles from source checkout or installed package data.""" + source_profiles = PROJECT_ROOT / "src" / "peachy" / "profiles" + if _is_source_checkout_root(PROJECT_ROOT) and source_profiles.is_dir(): + return source_profiles + + packaged_profiles = _packaged_profiles_directory() + if packaged_profiles is not None and packaged_profiles.is_dir(): + return packaged_profiles + + return source_profiles + + +DEFAULT_PROFILES_DIRECTORY = _resolve_default_profiles_directory() + +# UI-created profiles live under a writable instance dir +USER_PERSONALITIES_DIRNAME = "user_personalities" + +# Qwen3-TTS CustomVoice speaker catalog from the deployed Hugging Face backend. +HF_AVAILABLE_VOICES: list[str] = [ + "Aiden", + "Ryan", + "Dylan", + "Eric", + "Ono_Anna", + "Serena", + "Sohee", + "Uncle_Fu", + "Vivian", +] + +HF_BACKEND = "huggingface" +HF_REALTIME_CONNECTION_MODE_ENV = "HF_REALTIME_CONNECTION_MODE" +HF_REALTIME_WS_URL_ENV = "HF_REALTIME_WS_URL" +REALTIME_TRANSCRIPTION_LANGUAGE_ENV = "REALTIME_TRANSCRIPTION_LANGUAGE" +HF_LOCAL_CONNECTION_MODE = "local" +HF_DEPLOYED_CONNECTION_MODE = "deployed" +HF_REALTIME_SESSION_PROXY_URL = "https://pollen-robotics-reachy-mini-realtime-url.hf.space/session" + + +@dataclass(frozen=True) +class HFBackendDefaults: + """Defaults for the Hugging Face realtime backend.""" + + connection_mode: str = HF_DEPLOYED_CONNECTION_MODE + # App-managed Hugging Face Space proxy. The Space forwards to the current + # session allocator, so allocator changes do not require app releases. + # Users who need a custom target should use HF_REALTIME_CONNECTION_MODE=local + # with HF_REALTIME_WS_URL. + session_url: str = HF_REALTIME_SESSION_PROXY_URL + voice: str = "Aiden" + direct_port: int = 8765 + + +HF_DEFAULTS = HFBackendDefaults() + +logger = logging.getLogger(__name__) + +# Removed backend selectors kept in stale robot .env files: warn but ignore them. +_OBSOLETE_BACKEND_ENV_NAMES = ("BACKEND_PROVIDER", "MODEL_NAME") + + +def _warn_on_obsolete_backend_env() -> None: + """Warn when removed multi-backend selectors are still set; Hugging Face is the only backend.""" + present = [name for name in _OBSOLETE_BACKEND_ENV_NAMES if (os.getenv(name) or "").strip()] + if present: + logger.warning( + "Ignoring obsolete backend environment variable(s): %s. This app now uses the Hugging Face backend only.", + ", ".join(present), + ) + + +def _env_flag(name: str, default: bool = False) -> bool: + """Parse a boolean environment flag. + + Accepted truthy values: 1, true, yes, on + Accepted falsy values: 0, false, no, off + """ + raw = os.getenv(name) + if raw is None: + return default + + value = raw.strip().lower() + if value in {"1", "true", "yes", "on"}: + return True + if value in {"0", "false", "no", "off"}: + return False + + logger.warning("Invalid boolean value for %s=%r, using default=%s", name, raw, default) + return default + + +APP_TIMEOUT_MINUTES_ENV = "REACHY_MINI_APP_TIMEOUT_MINUTES" +DEFAULT_APP_TIMEOUT_MINUTES = 1440.0 + + +def resolve_app_timeout_minutes() -> float | None: + """Read the app inactivity timeout (minutes) from the environment; None means disabled.""" + raw_value = os.getenv(APP_TIMEOUT_MINUTES_ENV, "").strip() + if not raw_value: + return DEFAULT_APP_TIMEOUT_MINUTES + try: + timeout_minutes = float(raw_value) + except ValueError: + logger.warning("Ignoring invalid %s=%r; using default.", APP_TIMEOUT_MINUTES_ENV, raw_value) + return DEFAULT_APP_TIMEOUT_MINUTES + return timeout_minutes if timeout_minutes > 0 else None + + +def _normalize_hf_connection_mode(value: str | None) -> str | None: + """Normalize the Hugging Face connection mode, if explicitly configured.""" + candidate = (value or "").strip().lower() + if not candidate: + return None + + if candidate not in {HF_LOCAL_CONNECTION_MODE, HF_DEPLOYED_CONNECTION_MODE}: + logger.warning( + "Invalid %s=%r. Expected local or deployed.", + HF_REALTIME_CONNECTION_MODE_ENV, + value, + ) + return None + return candidate + + +def _normalize_transcription_language(value: str | None) -> str: + """Return the configured realtime transcription language.""" + candidate = (value or "").strip() + return candidate or "en" + + +@dataclass(frozen=True) +class HFConnectionSelection: + """Resolved Hugging Face connection mode and target availability.""" + + mode: str + has_target: bool + session_url: str | None = None + direct_ws_url: str | None = None + + +@dataclass(frozen=True) +class HFRealtimeURLParts: + """Parsed Hugging Face realtime URL components used by UI and client setup.""" + + base_url: str + websocket_base_url: str + connect_query: dict[str, str] + host: str | None + port: int | None + has_realtime_path: bool + + +def parse_hf_realtime_url(realtime_url: str) -> HFRealtimeURLParts: + """Parse a Hugging Face realtime URL into OpenAI-compatible client endpoints.""" + parsed = urlsplit(realtime_url) + scheme = parsed.scheme.lower() + if scheme not in {"ws", "wss", "http", "https"}: + raise ValueError( + "Expected Hugging Face realtime URL to start with ws://, wss://, http://, or https://, " + f"got: {realtime_url}" + ) + + path = parsed.path.rstrip("/") + has_realtime_path = path.endswith("/realtime") + if has_realtime_path: + base_path = path[: -len("/realtime")] + else: + base_path = path + + connect_query = {key: value for key, value in parse_qsl(parsed.query, keep_blank_values=True) if key != "model"} + http_scheme = "https" if scheme in {"wss", "https"} else "http" + websocket_scheme = "wss" if scheme in {"wss", "https"} else "ws" + base_url = urlunsplit((http_scheme, parsed.netloc, base_path, "", "")) + websocket_base_url = urlunsplit((websocket_scheme, parsed.netloc, base_path, "", "")) + return HFRealtimeURLParts( + base_url=base_url, + websocket_base_url=websocket_base_url, + connect_query=connect_query, + host=parsed.hostname, + port=parsed.port or HF_DEFAULTS.direct_port, + has_realtime_path=has_realtime_path, + ) + + +def parse_hf_direct_target(ws_url: str | None) -> tuple[str | None, int | None]: + """Extract host and port from a direct Hugging Face realtime URL.""" + if not ws_url: + return None, None + try: + parsed = parse_hf_realtime_url(ws_url) + return parsed.host, parsed.port + except Exception: + return None, None + + +def build_hf_direct_ws_url(host: str, port: int) -> str: + """Build the direct Hugging Face realtime websocket URL used by the app.""" + return f"ws://{host}:{port}/v1/realtime" + + +def _collect_profile_names(profiles_root: Path) -> set[str]: + """Return profile folder names from a profiles root directory.""" + if not profiles_root.exists() or not profiles_root.is_dir(): + return set() + return {p.name for p in profiles_root.iterdir() if p.is_dir()} + + +def _collect_tool_module_names(tools_root: Path) -> set[str]: + """Return tool module names from a tools directory.""" + if not tools_root.exists() or not tools_root.is_dir(): + return set() + ignored = {"__init__", "core_tools"} + return {p.stem for p in tools_root.glob("*.py") if p.is_file() and p.stem not in ignored} + + +def _raise_on_name_collisions( + *, + label: str, + external_root: Path, + internal_root: Path, + external_names: set[str], + internal_names: set[str], +) -> None: + """Raise with a clear message when external/internal names collide.""" + collisions = sorted(external_names & internal_names) + if not collisions: + return + + raise RuntimeError( + f"Config.__init__(): Ambiguous {label} names found in both external and built-in libraries: {collisions}. " + f"External {label} root: {external_root}. Built-in {label} root: {internal_root}. " + f"Please rename the conflicting external {label}(s) to continue." + ) + + +# Validate LOCKED_PROFILE at startup +if LOCKED_PROFILE is not None: + _profiles_dir = DEFAULT_PROFILES_DIRECTORY + _profile_path = _profiles_dir / LOCKED_PROFILE + _instructions_file = _profile_path / "instructions.txt" + if not _profile_path.is_dir(): + logger.critical("LOCKED_PROFILE %r does not exist in %s", LOCKED_PROFILE, _profiles_dir) + sys.exit(1) + if not _instructions_file.is_file(): + logger.critical("LOCKED_PROFILE %r has no instructions.txt", LOCKED_PROFILE) + sys.exit(1) + +_skip_dotenv = _env_flag("REACHY_MINI_SKIP_DOTENV", default=False) + +if _skip_dotenv: + logger.info("Skipping .env loading because REACHY_MINI_SKIP_DOTENV is set") +else: + # Locate .env file (search upward from current working directory) + dotenv_path = find_dotenv(usecwd=True) + + if dotenv_path: + # Load .env and override environment variables + load_dotenv(dotenv_path=dotenv_path, override=True) + logger.info(f"Configuration loaded from {dotenv_path}") + else: + logger.warning("No .env file found, using environment variables") + +_warn_on_obsolete_backend_env() + + +class Config: + """Configuration class for the conversation app.""" + + HF_REALTIME_CONNECTION_MODE = ( + _normalize_hf_connection_mode(os.getenv(HF_REALTIME_CONNECTION_MODE_ENV)) or HF_DEFAULTS.connection_mode + ) + # Deliberately ignore HF_REALTIME_SESSION_URL from the environment; the app-managed proxy is HF_DEFAULTS.session_url. + HF_REALTIME_SESSION_URL = HF_DEFAULTS.session_url + HF_REALTIME_WS_URL = os.getenv(HF_REALTIME_WS_URL_ENV) + REALTIME_TRANSCRIPTION_LANGUAGE = _normalize_transcription_language(os.getenv(REALTIME_TRANSCRIPTION_LANGUAGE_ENV)) + HF_TOKEN = os.getenv("HF_TOKEN") # Optional, falls back to hf auth login if not set + + logger.debug( + "HF mode: %s, HF session URL set: %s, HF direct URL set: %s", + HF_REALTIME_CONNECTION_MODE, + bool(HF_REALTIME_SESSION_URL and HF_REALTIME_SESSION_URL.strip()), + bool(HF_REALTIME_WS_URL and HF_REALTIME_WS_URL.strip()), + ) + + # Filesystem root containing profile directories, not a Python import path. + _profiles_directory_env = os.getenv("REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY") + PROFILES_DIRECTORY = Path(_profiles_directory_env) if _profiles_directory_env else DEFAULT_PROFILES_DIRECTORY + INSTANCE_PATH: Path | None = None # set at startup; writable home for UI-created profiles + _tools_directory_env = os.getenv("REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY") + TOOLS_DIRECTORY = Path(_tools_directory_env) if _tools_directory_env else None + AUTOLOAD_EXTERNAL_TOOLS = _env_flag("AUTOLOAD_EXTERNAL_TOOLS", default=False) + REACHY_MINI_CUSTOM_PROFILE = LOCKED_PROFILE or os.getenv("REACHY_MINI_CUSTOM_PROFILE") + + logger.debug(f"Custom Profile: {REACHY_MINI_CUSTOM_PROFILE}") + + def __init__(self) -> None: + """Initialize the configuration.""" + if self.REACHY_MINI_CUSTOM_PROFILE and self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + selected_profile_path = self.PROFILES_DIRECTORY / self.REACHY_MINI_CUSTOM_PROFILE + if not selected_profile_path.is_dir(): + available_profiles = sorted(_collect_profile_names(self.PROFILES_DIRECTORY)) + raise RuntimeError( + "Config.__init__(): Selected profile " + f"'{self.REACHY_MINI_CUSTOM_PROFILE}' was not found in external profiles root " + f"{self.PROFILES_DIRECTORY}. " + f"Available external profiles: {available_profiles}. " + "Either set 'REACHY_MINI_CUSTOM_PROFILE' to one of the available external profiles " + "or unset 'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' to use built-in profiles." + ) + + if self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + external_profiles = _collect_profile_names(self.PROFILES_DIRECTORY) + internal_profiles = _collect_profile_names(DEFAULT_PROFILES_DIRECTORY) + _raise_on_name_collisions( + label="profile", + external_root=self.PROFILES_DIRECTORY, + internal_root=DEFAULT_PROFILES_DIRECTORY, + external_names=external_profiles, + internal_names=internal_profiles, + ) + + if self.TOOLS_DIRECTORY is not None: + builtin_tools_root = Path(__file__).parent / "tools" + external_tools = _collect_tool_module_names(self.TOOLS_DIRECTORY) + internal_tools = _collect_tool_module_names(builtin_tools_root) + _raise_on_name_collisions( + label="tool", + external_root=self.TOOLS_DIRECTORY, + internal_root=builtin_tools_root, + external_names=external_tools, + internal_names=internal_tools, + ) + + if self.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + logger.warning( + "Environment variable 'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' is set. " + "Profiles (instructions.txt, ...) will be loaded from %s.", + self.PROFILES_DIRECTORY, + ) + else: + logger.info( + "'REACHY_MINI_EXTERNAL_PROFILES_DIRECTORY' is not set. Using built-in profiles from %s.", + DEFAULT_PROFILES_DIRECTORY, + ) + + if self.TOOLS_DIRECTORY is not None: + logger.warning( + "Environment variable 'REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY' is set. " + "External tools will be loaded from %s.", + self.TOOLS_DIRECTORY, + ) + else: + logger.info("'REACHY_MINI_EXTERNAL_TOOLS_DIRECTORY' is not set. Using built-in shared tools only.") + + def user_personalities_root(self) -> Path: + """Writable root for UI-created profiles.""" + base = self.INSTANCE_PATH if self.INSTANCE_PATH is not None else resolve_state_root() + return Path(base) / USER_PERSONALITIES_DIRNAME + + def resolve_profile_dir(self, profile: str) -> Path: + """On-disk directory for a profile selection.""" + head, _, tail = profile.partition("/") + if head == USER_PERSONALITIES_DIRNAME and tail: + return self.user_personalities_root() / tail + return self.PROFILES_DIRECTORY / profile + + +config = Config() + + +def refresh_runtime_config_from_env() -> None: + """Refresh mutable runtime config fields from the current environment.""" + _warn_on_obsolete_backend_env() + config.HF_REALTIME_CONNECTION_MODE = ( + _normalize_hf_connection_mode(os.getenv(HF_REALTIME_CONNECTION_MODE_ENV)) or HF_DEFAULTS.connection_mode + ) + # Deliberately ignore HF_REALTIME_SESSION_URL from the environment; the app-managed proxy is HF_DEFAULTS.session_url. + config.HF_REALTIME_SESSION_URL = HF_DEFAULTS.session_url + config.HF_REALTIME_WS_URL = os.getenv(HF_REALTIME_WS_URL_ENV) + config.REALTIME_TRANSCRIPTION_LANGUAGE = _normalize_transcription_language( + os.getenv(REALTIME_TRANSCRIPTION_LANGUAGE_ENV) + ) + config.HF_TOKEN = os.getenv("HF_TOKEN") + config.REACHY_MINI_CUSTOM_PROFILE = LOCKED_PROFILE or os.getenv("REACHY_MINI_CUSTOM_PROFILE") + + +def get_available_voices() -> list[str]: + """Return the curated Hugging Face voice list.""" + return list(HF_AVAILABLE_VOICES) + + +def get_default_voice() -> str: + """Return the default Hugging Face voice.""" + return HF_DEFAULTS.voice + + +def get_hf_session_url() -> str | None: + """Return the built-in Hugging Face session proxy URL, if any.""" + value = (getattr(config, "HF_REALTIME_SESSION_URL", None) or "").strip() + return value or None + + +def get_hf_direct_ws_url() -> str | None: + """Return the configured direct Hugging Face realtime URL, if any.""" + value = (getattr(config, "HF_REALTIME_WS_URL", None) or "").strip() + return value or None + + +def get_hf_connection_selection() -> HFConnectionSelection: + """Resolve the selected Hugging Face connection mode and whether it is usable.""" + session_url = get_hf_session_url() + direct_ws_url = get_hf_direct_ws_url() + mode = _normalize_hf_connection_mode(getattr(config, "HF_REALTIME_CONNECTION_MODE", None)) + if mode is None: + raise RuntimeError(f"{HF_REALTIME_CONNECTION_MODE_ENV} must be set to local or deployed.") + + target = direct_ws_url if mode == HF_LOCAL_CONNECTION_MODE else session_url + + return HFConnectionSelection( + mode=mode, + has_target=bool(target), + session_url=session_url, + direct_ws_url=direct_ws_url, + ) + + +def has_hf_realtime_target() -> bool: + """Return whether Hugging Face has a target for the selected mode.""" + return get_hf_connection_selection().has_target + + +def set_instance_path(instance_path: str | Path | None) -> None: + """Record the app instance dir so UI-created profiles persist outside package data.""" + config.INSTANCE_PATH = Path(instance_path) if instance_path else None + + +def set_custom_profile(profile: str | None) -> None: + """Update the selected custom profile at runtime and expose it via env. + + This ensures modules that read `config` and code that inspects the + environment see a consistent value. + """ + if LOCKED_PROFILE is not None: + return + try: + config.REACHY_MINI_CUSTOM_PROFILE = profile + except Exception as e: + logger.warning("Failed to update config profile: %s", e) + try: + if profile: + os.environ["REACHY_MINI_CUSTOM_PROFILE"] = profile + else: + # Remove to reflect default + os.environ.pop("REACHY_MINI_CUSTOM_PROFILE", None) + except Exception as e: + logger.warning("Failed to sync profile to environment: %s", e) diff --git a/src/peachy/console.py b/src/peachy/console.py new file mode 100644 index 0000000000000000000000000000000000000000..60cabe2e25d9f899092ce32c09a6afaf60eaa387 --- /dev/null +++ b/src/peachy/console.py @@ -0,0 +1,1005 @@ +"""Bidirectional local audio stream with optional web settings UI. + +If the selected backend is missing its required API key, a settings page is +served via the Reachy Mini Apps settings server so users can configure it. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +import os +import time +import asyncio +import logging +import tempfile +import threading +from typing import List, Optional +from pathlib import Path +from collections.abc import Callable, AsyncGenerator + +from reachy_mini import ReachyMini +from reachy_mini.media.media_manager import MediaBackend +from peachy.config import ( + LOCKED_PROFILE, + HF_REALTIME_WS_URL_ENV, + HF_LOCAL_CONNECTION_MODE, + HF_DEPLOYED_CONNECTION_MODE, + HF_REALTIME_CONNECTION_MODE_ENV, + config, + get_default_voice, + get_hf_session_url, + build_hf_direct_ws_url, + parse_hf_direct_target, + get_hf_connection_selection, + refresh_runtime_config_from_env, +) +from peachy.streaming import AdditionalOutputs +from peachy.audio.resampling import resample_audio +from peachy.startup_settings import read_startup_settings, write_startup_settings +from peachy.tools.core_tools import initialize_tools +from peachy.personality_routes import mount_personality_routes +from peachy.audio.startup_config import apply_audio_startup_config +from peachy.conversation_handler import ConversationHandler +from peachy.providers import ( + OPENAI_DEFAULT_MODEL, + OPENAI_MODEL_ENV, + OPENAI_PROVIDER, + PROVIDER_ENV, + ProviderDefinition, + available_provider_names, + get_provider_definition, + redact_configured_credentials, + redact_secrets, + resolve_provider_name, +) + + +try: + # FastAPI is provided by the Reachy Mini Apps runtime + from fastapi import FastAPI, Request, Response + from pydantic import BaseModel + from fastapi.responses import FileResponse, JSONResponse, StreamingResponse + from starlette.staticfiles import StaticFiles +except Exception: # pragma: no cover - only loaded when settings_app is used + FastAPI = object # type: ignore + FileResponse = object # type: ignore + JSONResponse = object # type: ignore + StreamingResponse = object # type: ignore + StaticFiles = object # type: ignore + BaseModel = object # type: ignore + Request = object # type: ignore + +logger = logging.getLogger(__name__) + +_SSE_KEEPALIVE_INTERVAL_SECONDS = 15.0 # below typical 60s proxy idle timeout +SETTINGS_API_PREFIX = "/api/v1" + + +def _detach_framework_root_routes(app: "FastAPI") -> None: + """Strip framework routes that would shadow the settings UI.""" + routes = getattr(app, "router", None) + routes = getattr(routes, "routes", None) if routes else getattr(app, "routes", None) + if routes is None: + return + survivors = [] + for route in routes: + path = getattr(route, "path", None) + is_catch_all = isinstance(path, str) and path.startswith("/{") and path.endswith(":path}") + if path in ("/", "/static") or is_catch_all: + logger.debug("detaching framework-provided route %r (%s)", path, type(route).__name__) + continue + survivors.append(route) + routes[:] = survivors + + +class ConversationEventBus: + """Thread-safe fan-out bus that delivers conversation activity events to SSE subscribers.""" + + MAX_QUEUE_SIZE = 64 + + def __init__(self) -> None: + """Initialize the bus.""" + # Each entry is (loop, queue); loop captured at subscribe() time for cross-thread delivery. + self._subscribers: list[tuple[asyncio.AbstractEventLoop, "asyncio.Queue[str]"]] = [] + self._lock = threading.Lock() + + def subscribe(self) -> tuple["asyncio.Queue[str]", Callable[[], None]]: + """Return a per-subscriber queue and its unsubscribe callback. Must be called from a coroutine.""" + loop = asyncio.get_running_loop() + queue: "asyncio.Queue[str]" = asyncio.Queue(maxsize=self.MAX_QUEUE_SIZE) + entry = (loop, queue) + with self._lock: + self._subscribers.append(entry) + + def unsubscribe() -> None: + with self._lock: + try: + self._subscribers.remove(entry) + except ValueError: + pass + + return queue, unsubscribe + + def publish(self, event: str) -> None: + """Broadcast to all subscribers via call_soon_threadsafe. Drops events for full or closed queues.""" + with self._lock: + snapshot = list(self._subscribers) + for loop, queue in snapshot: + try: + loop.call_soon_threadsafe(self._enqueue_safely, queue, event) + except RuntimeError: + pass # loop closed; subscriber will clean itself up on disconnect + + @staticmethod + def _enqueue_safely(queue: "asyncio.Queue[str]", event: str) -> None: + try: + queue.put_nowait(event) + except asyncio.QueueFull: + logger.debug("conversation event dropped (subscriber queue full): %s", event) + + +async def _conversation_events_stream( + bus: ConversationEventBus, + request: "Request", +) -> "AsyncGenerator[str, None]": + """Yield SSE lines for one subscriber; emits keep-alive comments during idle periods.""" + queue, unsubscribe = bus.subscribe() + try: + yield "retry: 2000\n\n" + yield "event: ready\ndata: connected\n\n" + + while True: + if await request.is_disconnected(): + return + try: + event = await asyncio.wait_for(queue.get(), timeout=_SSE_KEEPALIVE_INTERVAL_SECONDS) + except asyncio.TimeoutError: + yield ": keep-alive\n\n" + continue + yield f"event: activity\ndata: {event}\n\n" + finally: + unsubscribe() + + +LOCAL_PLAYER_BACKEND = ( + getattr(MediaBackend, "LOCAL", None) + or getattr(MediaBackend, "GSTREAMER", None) + or getattr(MediaBackend, "DEFAULT", None) +) + +HandlerFactory = Callable[[Optional[str]], ConversationHandler] +ProviderResolver = Callable[[], ProviderDefinition] + +LEGACY_STARTUP_ENV_NAMES = ( + "REACHY_MINI_CUSTOM_PROFILE", + "REACHY_MINI_VOICE_OVERRIDE", +) +BACKEND_RETRY_DELAY_SECONDS = 5.0 + + +class LocalStream: + """LocalStream using Reachy Mini's recorder/player.""" + + def __init__( + self, + handler: ConversationHandler, + robot: ReachyMini, + *, + settings_app: Optional[FastAPI] = None, + instance_path: Optional[str] = None, + handler_factory: HandlerFactory | None = None, + provider_resolver: ProviderResolver = get_provider_definition, + startup_voice: Optional[str] = None, + ): + """Initialize the stream with a realtime handler and pipelines. + + - ``settings_app``: the Reachy Mini Apps FastAPI to attach settings endpoints. + - ``instance_path``: directory where per-instance ``.env`` should be stored. + - ``handler_factory``: builds a fresh handler for the currently selected backend. + - ``provider_resolver``: returns the current provider definition and readiness. + """ + self._robot = robot + self._stop_event = asyncio.Event() + self._restart_requested = asyncio.Event() + self._tasks: List[asyncio.Task[None]] = [] + self._handler_factory = handler_factory + self._provider_resolver = provider_resolver + self._voice_override = startup_voice + self._settings_app: Optional[FastAPI] = settings_app + self._instance_path: Optional[str] = instance_path + self._settings_initialized = False + self._asyncio_loop = None + self._mic_muted = False # mic starts live; the UI toggles it via the settings API + self._backend_connection_state = "not_started" + self._backend_error: str | None = None + self._backend_retry_delay = BACKEND_RETRY_DELAY_SECONDS + # One bus for the stream's lifetime: the conversation events route + # closes over it, so handler rebuilds must keep publishing into it. + self._event_bus = ConversationEventBus() + self._install_handler(handler) + + def _install_handler(self, handler: ConversationHandler) -> None: + """Set the active handler and wire LocalStream-owned helpers into it.""" + self.handler = handler + self.handler._clear_queue = self.clear_audio_queue + self._attach_event_bus_to_handler() + + def _attach_event_bus_to_handler(self) -> None: + """Wire the event bus as the handler's activity observer, if supported.""" + setter = getattr(self.handler, "set_activity_observer", None) + if callable(setter): + setter(self._event_bus.publish) + + def seconds_since_activity(self) -> float: + """Seconds since the live handler last saw conversation activity.""" + return time.monotonic() - self.handler.last_activity_time + + def _read_env_lines(self, env_path: Path) -> list[str]: + """Load env file contents or a template as a list of lines.""" + inst = env_path.parent + try: + if env_path.exists(): + try: + return env_path.read_text(encoding="utf-8").splitlines() + except Exception: + return [] + template_text = None + ex = inst / ".env.example" + if ex.exists(): + try: + template_text = ex.read_text(encoding="utf-8") + except Exception: + template_text = None + if template_text is None: + try: + cwd_example = Path.cwd() / ".env.example" + if cwd_example.exists(): + template_text = cwd_example.read_text(encoding="utf-8") + except Exception: + template_text = None + if template_text is None: + packaged = Path(__file__).parent / ".env.example" + if packaged.exists(): + try: + template_text = packaged.read_text(encoding="utf-8") + except Exception: + template_text = None + return template_text.splitlines() if template_text else [] + except Exception: + return [] + + def _backend_connected(self) -> bool: + """Return whether the active handler currently has a realtime connection.""" + try: + handler_state = vars(self.handler) + except TypeError: + handler_state = {} + return handler_state.get("connection") is not None + + def _can_rebuild_handler(self) -> bool: + """Return whether LocalStream can construct handlers for backend changes.""" + return self._handler_factory is not None + + def _build_handler_for_current_backend(self) -> ConversationHandler: + """Create and install a fresh handler for the current runtime backend config.""" + if self._handler_factory is None: + return self.handler + handler = self._handler_factory(self._voice_override) + self._install_handler(handler) + return handler + + async def _shutdown_active_handler(self) -> None: + """Best-effort shutdown for the currently active handler.""" + try: + await self.handler.shutdown() + except asyncio.CancelledError: + raise + except Exception as e: + logger.debug( + "Active handler shutdown ignored during restart: %s", + redact_configured_credentials(e), + ) + + def _mark_restart_requested(self, reason: str) -> None: + """Request a backend restart from a synchronous route handler.""" + logger.info("Backend restart requested: %s", reason) + self._set_backend_connection_state("connecting") + loop = self._asyncio_loop + if loop is not None and loop.is_running(): + asyncio.run_coroutine_threadsafe(self.request_backend_restart(reason), loop) + return + self._restart_requested.set() + + async def request_backend_restart(self, reason: str) -> None: + """Ask the startup loop to rebuild the backend and stop the current handler.""" + self._set_backend_connection_state("connecting") + self._restart_requested.set() + self.clear_audio_queue() + await self._shutdown_active_handler() + + async def _sleep_or_restart_requested(self, delay: float) -> None: + """Sleep for a retry interval, waking early if a restart is requested.""" + if self._restart_requested.is_set(): + return + try: + await asyncio.wait_for(self._restart_requested.wait(), timeout=delay) + except asyncio.TimeoutError: + pass + + async def _wait_for_restart_or_stop(self) -> None: + """Wait until configuration/manual restart or app shutdown.""" + while not self._stop_event.is_set() and not self._restart_requested.is_set(): + await asyncio.sleep(0.1) + + @staticmethod + def _format_backend_error(error: BaseException | str) -> str: + """Return a compact user-facing backend error string.""" + if isinstance(error, str): + return redact_secrets(error) + message = str(error).strip() + rendered = f"{type(error).__name__}: {message}" if message else type(error).__name__ + return redact_secrets(rendered) + + def _set_backend_connection_state(self, state: str, error: BaseException | str | None = None) -> None: + """Update backend connection status exposed through the settings UI.""" + self._backend_connection_state = state + if error is not None: + self._backend_error = self._format_backend_error(error) + elif state != "disconnected": + self._backend_error = None + + def _backend_connection_status(self) -> dict[str, object]: + """Return the backend connection state exposed in the settings API.""" + connected = self._backend_connected() + state = "connected" if connected else self._backend_connection_state + return { + "backend_connected": connected, + "backend_connection_state": state, + "backend_error": None if connected else self._backend_error, + } + + @staticmethod + def _apply_runtime_env_values( + updates: dict[str, str], + removals: tuple[str, ...] = (), + ) -> None: + """Apply runtime environment updates, rolling back on refresh failure.""" + affected_names = set(updates).union(removals) + previous_values = {name: os.environ.get(name) for name in affected_names} + try: + for env_name in removals: + os.environ.pop(env_name, None) + for env_name, value in updates.items(): + os.environ[env_name] = value + refresh_runtime_config_from_env() + except Exception: + for env_name, previous in previous_values.items(): + if previous is None: + os.environ.pop(env_name, None) + else: + os.environ[env_name] = previous + try: + refresh_runtime_config_from_env() + except Exception: + pass + raise + + @staticmethod + def _write_env_file_atomically(env_path: Path, text: str) -> None: + """Securely replace an env file without exposing a partial credential write.""" + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{env_path.name}.", + suffix=".tmp", + dir=env_path.parent, + ) + os.close(file_descriptor) + temporary_path = Path(temporary_name) + try: + temporary_path.write_text(text, encoding="utf-8") + temporary_path.chmod(0o600) + temporary_path.replace(env_path) + finally: + temporary_path.unlink(missing_ok=True) + + def _persist_env_values( + self, + updates: dict[str, str], + *, + removals: tuple[str, ...] = (), + ) -> None: + """Persist environment updates/removals transactionally, then refresh runtime config.""" + normalized_updates = {name: (value or "").strip() for name, value in updates.items()} + normalized_updates = {name: value for name, value in normalized_updates.items() if value} + normalized_removals = tuple( + sorted( + { + name.strip() + for name in removals + if name and name.strip() and name.strip() not in normalized_updates + } + ) + ) + if not normalized_updates and not normalized_removals: + return + + if not self._instance_path: + self._apply_runtime_env_values(normalized_updates, normalized_removals) + return + + inst = Path(self._instance_path) + env_path = inst / ".env" + original_exists = env_path.exists() + original_text = env_path.read_text(encoding="utf-8") if original_exists else None + lines = original_text.splitlines() if original_text is not None else self._read_env_lines(env_path) + lines = [ + line + for line in lines + if not any(line.strip().startswith(f"{env_name}=") for env_name in normalized_removals) + ] + for env_name, value in normalized_updates.items(): + replaced = False + for i, line in enumerate(lines): + if line.strip().startswith(f"{env_name}="): + lines[i] = f"{env_name}={value}" + replaced = True + break + if not replaced: + lines.append(f"{env_name}={value}") + final_text = "\n".join(lines) + "\n" + + self._write_env_file_atomically(env_path, final_text) + try: + self._apply_runtime_env_values(normalized_updates, normalized_removals) + except Exception: + try: + if original_exists and original_text is not None: + self._write_env_file_atomically(env_path, original_text) + else: + env_path.unlink(missing_ok=True) + except Exception: + logger.error("Failed to roll back provider configuration file after runtime update failure") + raise + changed_names = sorted((*normalized_updates, *normalized_removals)) + logger.info("Persisted %s to %s", ", ".join(changed_names), env_path) + + def _remove_persisted_env_values(self, env_names: tuple[str, ...]) -> None: + """Remove keys from the instance `.env` without mutating the current runtime.""" + normalized_names = tuple(sorted({name.strip() for name in env_names if name and name.strip()})) + if not normalized_names or not self._instance_path: + return + + env_path = Path(self._instance_path) / ".env" + if not env_path.exists(): + return + + try: + lines = env_path.read_text(encoding="utf-8").splitlines() + filtered_lines = [ + line + for line in lines + if not any(line.strip().startswith(f"{env_name}=") for env_name in normalized_names) + ] + if filtered_lines == lines: + return + + final_text = "\n".join(filtered_lines) + if final_text: + final_text += "\n" + env_path.write_text(final_text, encoding="utf-8") + env_path.chmod(0o600) + logger.info("Removed %s from %s", ", ".join(normalized_names), env_path) + except Exception as e: + logger.warning("Failed to remove %s: %s", ", ".join(normalized_names), e) + + def _persist_hf_direct_connection(self, host: str, port: int) -> None: + """Persist a direct Hugging Face websocket target.""" + self._persist_env_values( + { + HF_REALTIME_CONNECTION_MODE_ENV: HF_LOCAL_CONNECTION_MODE, + HF_REALTIME_WS_URL_ENV: build_hf_direct_ws_url(host, port), + } + ) + + def _persist_hf_allocator_connection(self) -> None: + """Persist the deployed Hugging Face allocator mode.""" + self._persist_env_values( + {HF_REALTIME_CONNECTION_MODE_ENV: HF_DEPLOYED_CONNECTION_MODE}, + removals=("HF_REALTIME_SESSION_URL",), + ) + + def _persist_personality(self, profile: Optional[str], voice_override: Optional[str] = None) -> None: + """Persist startup profile and voice in instance-local UI settings.""" + if LOCKED_PROFILE is not None: + return + selection = (profile or "").strip() or None + normalized_voice_override = (voice_override or "").strip() or None + try: + from peachy.config import set_custom_profile + + set_custom_profile(selection) + except Exception: + pass + + if not self._instance_path: + return + try: + write_startup_settings( + self._instance_path, + profile=selection, + voice=normalized_voice_override, + ) + self._remove_persisted_env_values(LEGACY_STARTUP_ENV_NAMES) + logger.info("Persisted startup personality settings to %s", Path(self._instance_path)) + except Exception as e: + logger.warning("Failed to persist startup personality settings: %s", e) + + def _read_persisted_personality(self) -> Optional[str]: + """Read the saved startup personality from instance-local UI settings.""" + return read_startup_settings(self._instance_path).profile + + async def apply_personality(self, profile: Optional[str]) -> str: + """Apply a personality by updating config and restarting the active backend.""" + try: + from peachy.config import set_custom_profile + from peachy.prompts import get_session_voice, get_session_instructions + + previous_profile = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None) + set_custom_profile(profile) + try: + get_session_instructions() + get_session_voice(default=get_default_voice()) + except Exception: + set_custom_profile(previous_profile) + raise + except Exception as e: + safe_error = redact_secrets(e) + logger.error("Error applying personality '%s': %s", profile, safe_error) + return f"Failed to apply personality: {safe_error}" + + # Rebuild the tool registry + initialize_tools(force=True) + await self.request_backend_restart("personality_changed") + return "Applied personality and restarting backend." + + async def get_available_voices(self) -> list[str]: + """Return voices exposed by the active provider handler.""" + try: + return list(await self.handler.get_available_voices()) + except asyncio.CancelledError: + raise + except Exception as e: + logger.error("Failed to read voices from active provider: %s", redact_configured_credentials(e)) + return [] + + def get_current_voice(self) -> str: + """Return the active provider handler's current voice.""" + try: + handler_voice = self.handler.get_current_voice() + if isinstance(handler_voice, str): + return handler_voice + except Exception as e: + logger.error("Failed to read current voice from active provider: %s", redact_configured_credentials(e)) + + # Preserve the pre-provider fallback for incomplete test/external handlers. + if self._voice_override: + return self._voice_override + try: + from peachy.prompts import get_session_voice + + return get_session_voice(default=get_default_voice()) + except Exception: + return get_default_voice() + + async def change_voice(self, voice: str) -> str: + """Change the voice through the active handler without rebuilding the backend.""" + try: + status = await self.handler.change_voice(voice) + except asyncio.CancelledError: + raise + except Exception as e: + safe_error = redact_configured_credentials(e) + logger.error("Error changing voice to %r: %s", voice, safe_error) + return f"Failed to change voice: {safe_error}" + + try: + current_voice = self.handler.get_current_voice() + if isinstance(current_voice, str) and current_voice.strip(): + self._voice_override = current_voice + except Exception as e: + logger.debug( + "Could not sync LocalStream voice override after voice change: %s", + redact_configured_credentials(e), + ) + if self._voice_override: + self._persist_voice_override(self._voice_override) + return status + + def _persist_voice_override(self, voice: str) -> None: + """Persist the chosen voice as the startup voice, keeping the startup profile.""" + if not self._instance_path: + return + try: + existing = read_startup_settings(self._instance_path) + write_startup_settings(self._instance_path, profile=existing.profile, voice=voice) + except Exception as e: + logger.warning("Failed to persist startup voice: %s", e) + + def _init_settings_ui_if_needed(self) -> None: + """Attach minimal settings UI to the settings app. + + Always mounts the UI when a settings_app is provided so that users + see a confirmation message even if the API key is already configured. + """ + if self._settings_initialized: + return + if self._settings_app is None: + return + settings_app = self._settings_app + + static_dir = Path(__file__).parent / "static" + index_file = static_dir / "index.html" + logger.info("Serving settings UI from %s", static_dir) + + # Framework pre-registers GET / and /static; strip them so our routes aren't shadowed. + _detach_framework_root_routes(settings_app) + + if hasattr(settings_app, "mount"): + try: + settings_app.mount("/static", StaticFiles(directory=str(static_dir)), name="static") + except Exception: + pass + + class BackendPayload(BaseModel): + provider: Optional[str] = None + openai_api_key: Optional[str] = None + openai_model: Optional[str] = None + hf_mode: Optional[str] = None + hf_host: Optional[str] = None + hf_port: Optional[int] = None + + def _status_payload() -> dict[str, object]: + definition = self._provider_resolver() + readiness = definition.readiness() + backend_connection = self._backend_connection_status() + payload = { + "provider": definition.name, + "available_providers": list(available_provider_names()), + "provider_state": readiness.state, + "provider_message": readiness.message, + "can_proceed": readiness.state == "ready", + "requires_restart": not self._can_rebuild_handler(), + **definition.redacted_status(), + **backend_connection, + } + return { + name: redact_secrets(value) if isinstance(value, str) else value + for name, value in payload.items() + } + + # GET / -> index.html + @settings_app.get("/") + def _root() -> FileResponse: + return FileResponse(str(index_file)) + + # GET /favicon.ico -> optional, avoid noisy 404s on some browsers + @settings_app.get("/favicon.ico") + def _favicon() -> Response: + return Response(status_code=204) + + @settings_app.get(f"{SETTINGS_API_PREFIX}/status") + def _status() -> JSONResponse: + return JSONResponse(_status_payload()) + + @settings_app.post(f"{SETTINGS_API_PREFIX}/reconnect") + def _reconnect() -> JSONResponse: + self._mark_restart_requested("manual_reconnect") + return JSONResponse({"ok": True, "message": "Reconnecting selected provider."}) + + event_bus = self._event_bus + + @settings_app.get(f"{SETTINGS_API_PREFIX}/conversation_events") + async def _conversation_events(request: Request) -> StreamingResponse: + return StreamingResponse( + _conversation_events_stream(event_bus, request), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", # disable proxy buffering (e.g. nginx) + "Connection": "keep-alive", + }, + ) + + class MicPayload(BaseModel): + muted: bool + + @settings_app.get(f"{SETTINGS_API_PREFIX}/mic") + def _mic_state() -> JSONResponse: + return JSONResponse({"muted": self._mic_muted}) + + @settings_app.post(f"{SETTINGS_API_PREFIX}/mic") + def _set_mic(payload: MicPayload) -> JSONResponse: + self._mic_muted = bool(payload.muted) + logger.info("Microphone %s via web UI", "muted" if self._mic_muted else "unmuted") + return JSONResponse({"muted": self._mic_muted}) + + @settings_app.post(f"{SETTINGS_API_PREFIX}/backend_config") + def _set_backend(payload: BackendPayload) -> JSONResponse: + selected = resolve_provider_name(payload.provider) + if selected not in available_provider_names(): + return JSONResponse({"ok": False, "error": "invalid_provider"}, status_code=400) + + updates = {PROVIDER_ENV: selected} + removals: tuple[str, ...] = () + if selected == OPENAI_PROVIDER: + supplied_key = (payload.openai_api_key or "").strip() + existing_key = (os.getenv("OPENAI_API_KEY") or "").strip() + if not supplied_key and not existing_key: + return JSONResponse({"ok": False, "error": "empty_key"}, status_code=400) + if supplied_key: + updates["OPENAI_API_KEY"] = supplied_key + model = (payload.openai_model or os.getenv(OPENAI_MODEL_ENV) or OPENAI_DEFAULT_MODEL).strip() + updates[OPENAI_MODEL_ENV] = model or OPENAI_DEFAULT_MODEL + else: + hf_selection = get_hf_connection_selection() + hf_mode = (payload.hf_mode or hf_selection.mode).strip().lower() + if hf_mode == HF_LOCAL_CONNECTION_MODE: + existing_host, existing_port = parse_hf_direct_target(hf_selection.direct_ws_url) + host = (payload.hf_host or "").strip() or existing_host or "" + if not host: + return JSONResponse({"ok": False, "error": "empty_hf_host"}, status_code=400) + if "://" in host or "/" in host or "?" in host or "#" in host: + return JSONResponse({"ok": False, "error": "invalid_hf_host"}, status_code=400) + + port = payload.hf_port if payload.hf_port is not None else existing_port or 8765 + if port < 1 or port > 65535: + return JSONResponse({"ok": False, "error": "invalid_hf_port"}, status_code=400) + + updates[HF_REALTIME_CONNECTION_MODE_ENV] = HF_LOCAL_CONNECTION_MODE + updates[HF_REALTIME_WS_URL_ENV] = build_hf_direct_ws_url(host, port) + elif hf_mode == HF_DEPLOYED_CONNECTION_MODE: + if not bool(get_hf_session_url()): + return JSONResponse({"ok": False, "error": "missing_hf_session_url"}, status_code=400) + updates[HF_REALTIME_CONNECTION_MODE_ENV] = HF_DEPLOYED_CONNECTION_MODE + removals = ("HF_REALTIME_SESSION_URL",) + else: + return JSONResponse({"ok": False, "error": "invalid_hf_mode"}, status_code=400) + + try: + self._persist_env_values(updates, removals=removals) + except Exception as e: + logger.warning("Failed to persist provider configuration (%s)", type(e).__name__) + return JSONResponse({"ok": False, "error": "persistence_failed"}, status_code=500) + if self._can_rebuild_handler(): + self._mark_restart_requested("provider_config_changed") + message = "Connection saved. Reconnecting backend." + else: + message = "Connection saved. Restart Reachy Mini Conversation from the desktop app to apply it." + return JSONResponse( + { + "ok": True, + "message": message, + **_status_payload(), + } + ) + + try: + mount_personality_routes( + self._settings_app, + self.handler, + lambda: self._asyncio_loop, + persist_personality=self._persist_personality, + get_persisted_personality=self._read_persisted_personality, + apply_personality=self.apply_personality, + get_voices=self.get_available_voices, + get_current_voice=self.get_current_voice, + change_voice=self.change_voice, + api_prefix=SETTINGS_API_PREFIX, + ) + except Exception: + logger.exception("Failed to mount personality routes; the personality UI will be unavailable") + + self._settings_initialized = True + + async def _run_handler_startup_loop(self) -> None: + """Start the realtime handler and keep settings UI alive after backend failures.""" + while not self._stop_event.is_set(): + definition = self._provider_resolver() + readiness = definition.readiness() + if readiness.state != "ready": + self._set_backend_connection_state(readiness.state, readiness.message) + await self._wait_for_restart_or_stop() + if self._stop_event.is_set(): + return + + if self._restart_requested.is_set(): + await self._shutdown_active_handler() + self._restart_requested.clear() + if not self._can_rebuild_handler(): + self._set_backend_connection_state("restart_required") + await self._wait_for_restart_or_stop() + continue + try: + self._build_handler_for_current_backend() + except Exception as exc: + self._set_backend_connection_state("configuration_error", exc) + await self._wait_for_restart_or_stop() + continue + definition = self._provider_resolver() + readiness = definition.readiness() + if readiness.state != "ready": + continue + + self._set_backend_connection_state("connecting") + try: + await self.handler.start_up() + except asyncio.CancelledError: + raise + except Exception as exc: + self._set_backend_connection_state("disconnected", exc) + logger.warning("Provider activation stopped: %s", redact_secrets(exc)) + else: + if self._stop_event.is_set(): + return + self._set_backend_connection_state("disconnected") + + if definition.auto_reconnect: + await self._sleep_or_restart_requested(self._backend_retry_delay) + else: + await self._wait_for_restart_or_stop() + + def launch(self) -> None: + """Start the recorder/player and run the async processing loops. + + If the selected backend is missing its required key, expose a tiny + settings UI via the Reachy Mini settings server to collect it before + starting streams. + """ + self._stop_event.clear() + + # Try to load an existing instance .env first (covers subsequent runs) + if self._instance_path: + try: + from dotenv import load_dotenv + + env_path = Path(self._instance_path) / ".env" + if env_path.exists(): + load_dotenv(dotenv_path=str(env_path), override=True) + refresh_runtime_config_from_env() + except Exception: + pass # Instance .env loading is optional; continue with defaults + + # Always expose settings UI if a settings app is available + # (do this AFTER loading the instance .env so status endpoint sees the right value) + self._init_settings_ui_if_needed() + + # Media starts even while the provider waits for configuration so the UI stays responsive. + self._robot.media.start_recording() + self._robot.media.start_playing() + + async def runner() -> None: + # Capture loop for cross-thread personality actions + loop = asyncio.get_running_loop() + self._asyncio_loop = loop # type: ignore[assignment] + # Connect the backend first so it overlaps the warmup and audio config below. + handler_task = asyncio.create_task(self._run_handler_startup_loop(), name="realtime-handler") + self._tasks = [handler_task] + await asyncio.sleep(1) # give the pipelines time to start + await asyncio.to_thread(apply_audio_startup_config, self._robot, logger=logger) + self._tasks += [ + asyncio.create_task(self.record_loop(), name="stream-record-loop"), + asyncio.create_task(self.play_loop(), name="stream-play-loop"), + ] + try: + await asyncio.gather(*self._tasks) + except asyncio.CancelledError: + logger.info("Tasks cancelled during shutdown") + finally: + # Ensure handler connection is closed + await self._shutdown_active_handler() + + asyncio.run(runner()) + + def close(self) -> None: + """Stop the stream and underlying media pipelines. + + This method: + - Stops audio recording and playback first + - Sets the stop event to signal async loops to terminate + - Cancels all pending async tasks (openai-handler, record-loop, play-loop) + """ + logger.info("Stopping LocalStream...") + + # Stop media pipelines FIRST before cancelling async tasks + # This ensures clean shutdown before PortAudio cleanup + try: + self._robot.media.stop_recording() + except Exception as e: + logger.debug(f"Error stopping recording (may already be stopped): {e}") + + try: + self._robot.media.stop_playing() + except Exception as e: + logger.debug(f"Error stopping playback (may already be stopped): {e}") + + # close() runs on watcher threads, loop-owned state must change on the loop. + loop = self._asyncio_loop + if loop is None or not loop.is_running(): + self._stop_event.set() + return + loop.call_soon_threadsafe(self._stop_event.set) + for task in self._tasks: + if not task.done(): + loop.call_soon_threadsafe(task.cancel) + + def clear_audio_queue(self) -> None: + """Flush queued playback audio immediately on user barge-in. + + Calls the SDK's ``clear_player()`` — now a first-class flush on both + the local GStreamer and WebRTC backends (the WebRTC one also tells the + daemon to drop audio already queued for the speaker). Falls back to the + deprecated ``clear_output_buffer()`` only for older SDKs. + """ + logger.info("User intervention: flushing player queue") + audio = getattr(self._robot.media, "audio", None) + if audio is not None: + if hasattr(audio, "clear_player") and callable(audio.clear_player): + audio.clear_player() + elif hasattr(audio, "clear_output_buffer") and callable(audio.clear_output_buffer): + # Older SDK without clear_player(); best-effort. + audio.clear_output_buffer() + # Drain the handler's pending output in place — do NOT replace the + # queue object, since emit() may be awaiting it (wait_for_item). + self._drain_output_queue() + + def _drain_output_queue(self) -> None: + """Empty the handler's output queue in place without replacing it.""" + queue = getattr(self.handler, "output_queue", None) + if queue is None: + return + while not queue.empty(): + try: + queue.get_nowait() + except asyncio.QueueEmpty: + break + + async def record_loop(self) -> None: + """Read mic frames from the recorder and forward them to the handler.""" + input_sample_rate = self._robot.media.get_input_audio_samplerate() + logger.debug(f"Audio recording started at {input_sample_rate} Hz") + + while not self._stop_event.is_set(): + audio_frame = self._robot.media.get_audio_sample() + if audio_frame is not None and not self._mic_muted: + await self.handler.receive((input_sample_rate, audio_frame)) + await asyncio.sleep(0) # avoid busy loop + + async def play_loop(self) -> None: + """Fetch outputs from the handler: log text and play audio frames.""" + while not self._stop_event.is_set(): + handler = self.handler + try: + handler_output = await asyncio.wait_for(handler.emit(), timeout=0.5) + except asyncio.TimeoutError: + continue + + if isinstance(handler_output, AdditionalOutputs): + for msg in handler_output.args: + content = msg.get("content", "") + if isinstance(content, str): + logger.info( + "role=%s content=%s", + msg.get("role"), + content if len(content) < 500 else content[:500] + "…", + ) + + elif isinstance(handler_output, tuple): + source_rate, audio_data = handler_output + if audio_data.size == 0: + continue + output_rate = self._robot.media.get_output_audio_samplerate() + audio_frame = resample_audio(audio_data, source_rate, output_rate) + self._robot.media.push_audio_sample(audio_frame) + + else: + logger.debug("Ignoring output type=%s", type(handler_output).__name__) + + await asyncio.sleep(0) # yield to event loop diff --git a/src/peachy/conversation_handler.py b/src/peachy/conversation_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..d56a0529c06f722bad0a0bf9d272aff4db2d4261 --- /dev/null +++ b/src/peachy/conversation_handler.py @@ -0,0 +1,133 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +from __future__ import annotations +import time +import asyncio +import logging +from abc import ABC, abstractmethod +from typing import ClassVar, TypeAlias +from collections.abc import Callable + +from peachy.streaming import AdditionalOutputs, AsyncStreamHandler, AudioArray, wait_for_item +from peachy.idle_policy import start_idle_tool_call +from peachy.tools.core_tools import ToolDependencies, get_tool_specs +from peachy.tools.background_tool_manager import BackgroundToolManager + + +logger = logging.getLogger(__name__) + + +AudioFrame: TypeAlias = tuple[int, AudioArray] +HandlerOutput: TypeAlias = AudioFrame | AdditionalOutputs | None +QueueItem: TypeAlias = AudioFrame | AdditionalOutputs + + +class ConversationHandler(AsyncStreamHandler, ABC): + """Shared app handler contract and idle behavior for realtime conversation backends.""" + + IDLE_BEHAVIOR_THRESHOLD_S: ClassVar[float] = 180.0 + + deps: ToolDependencies + tool_manager: BackgroundToolManager + output_queue: asyncio.Queue[QueueItem] + last_activity_time: float + last_idle_behavior_time: float + _activity_observer: Callable[[str], None] | None = None + + def __init__(self) -> None: + """Initialize the stream handler and shared idle/activity tracking.""" + super().__init__() + self.last_activity_time = time.monotonic() + self.last_idle_behavior_time = self.last_activity_time + + def set_activity_observer(self, observer: Callable[[str], None] | None) -> None: + """Attach or detach an activity observer. Pass None to clear.""" + self._activity_observer = observer + + def _mark_activity(self, reason: str) -> None: + """Record non-idle conversation activity for the idle timer.""" + self.last_activity_time = time.monotonic() + logger.debug("last activity time updated to %s (%s)", self.last_activity_time, reason) + if self._activity_observer is not None: + try: + self._activity_observer(reason) + except Exception: + logger.debug("activity observer raised (ignored)", exc_info=True) + + def _idle_behavior_ready(self) -> bool: + """Return whether idle behavior may run now. Backends can add guards.""" + return True + + async def emit(self) -> HandlerOutput: + """Emit the next queued output, triggering local idle behavior when due.""" + now = time.monotonic() + idle_duration = now - self.last_activity_time + idle_behavior_duration = now - self.last_idle_behavior_time + if ( + idle_duration > self.IDLE_BEHAVIOR_THRESHOLD_S + and idle_behavior_duration > self.IDLE_BEHAVIOR_THRESHOLD_S + and self._idle_behavior_ready() + and self.deps.movement_manager.is_idle() + ): + try: + await self.send_idle_signal(idle_duration) + except Exception as e: + logger.warning("Idle tool skipped (connection closed?): %s", e) + return None + self.last_idle_behavior_time = now + handler_output = await wait_for_item(self.output_queue) + return handler_output + + async def send_idle_signal(self, idle_duration: float) -> None: + """Run a locally selected idle tool without sending an idle turn to the model.""" + if not self._is_connected(): + logger.debug("No active session; cannot run idle tool") + return + + available_tool_names = {spec["name"] for spec in get_tool_specs()} + await start_idle_tool_call( + deps=self.deps, + tool_manager=self.tool_manager, + output_queue=self.output_queue, + available_tool_names=available_tool_names, + idle_duration=idle_duration, + ) + + @abstractmethod + def _is_connected(self) -> bool: + """Return whether the backend session/connection is currently open.""" + ... + + @abstractmethod + async def start_up(self) -> None: + """Start the realtime handler.""" + ... + + @abstractmethod + async def shutdown(self) -> None: + """Shut down the realtime handler.""" + ... + + @abstractmethod + async def receive(self, frame: AudioFrame) -> None: + """Receive an input audio frame.""" + ... + + @abstractmethod + async def apply_personality(self, profile: str | None) -> str: + """Apply a personality profile.""" + ... + + @abstractmethod + async def get_available_voices(self) -> list[str]: + """Return voices available for the active backend.""" + ... + + @abstractmethod + def get_current_voice(self) -> str: + """Return the current voice.""" + ... + + @abstractmethod + async def change_voice(self, voice: str) -> str: + """Change the current voice.""" + ... diff --git a/src/peachy/dance_emotion_moves.py b/src/peachy/dance_emotion_moves.py new file mode 100644 index 0000000000000000000000000000000000000000..0641c4b7aadcd7f9b54602a4be327f2f27866548 --- /dev/null +++ b/src/peachy/dance_emotion_moves.py @@ -0,0 +1,156 @@ +"""Dance and emotion moves for the movement queue system. + +This module implements dance moves and emotions as Move objects that can be queued +and executed sequentially by the MovementManager. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import logging +from typing import Tuple + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini.motion.move import Move +from reachy_mini.motion.recorded_move import RecordedMoves +from reachy_mini_dances_library.dance_move import DanceMove + + +logger = logging.getLogger(__name__) + + +class DanceQueueMove(Move): # type: ignore + """Wrapper for dance moves to work with the movement queue system.""" + + def __init__(self, move_name: str): + """Initialize a DanceQueueMove.""" + self.dance_move = DanceMove(move_name) + self.move_name = move_name + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float(self.dance_move.duration) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate dance move at time t.""" + try: + # Get the pose from the dance move + head_pose, antennas, body_yaw = self.dance_move.evaluate(t) + + # Convert to numpy array if antennas is tuple and return in official Move format + if isinstance(antennas, tuple): + antennas = np.array([antennas[0], antennas[1]]) + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating dance move '{self.move_name}' at t={t}: {e}") + # Return neutral pose on error + from reachy_mini.utils import create_head_pose + + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0) + + +class EmotionQueueMove(Move): # type: ignore + """Wrapper for emotion moves to work with the movement queue system.""" + + def __init__(self, emotion_name: str, recorded_moves: RecordedMoves): + """Initialize an EmotionQueueMove.""" + self.emotion_move = recorded_moves.get(emotion_name) + self.emotion_name = emotion_name + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float(self.emotion_move.duration) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate emotion move at time t.""" + try: + # Get the pose from the emotion move + head_pose, antennas, body_yaw = self.emotion_move.evaluate(t) + + # Convert to numpy array if antennas is tuple and return in official Move format + if isinstance(antennas, tuple): + antennas = np.array([antennas[0], antennas[1]]) + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating emotion '{self.emotion_name}' at t={t}: {e}") + # Return neutral pose on error + from reachy_mini.utils import create_head_pose + + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + return (neutral_head_pose, np.array([0.0, 0.0], dtype=np.float64), 0.0) + + +class GotoQueueMove(Move): # type: ignore + """Wrapper for goto moves to work with the movement queue system.""" + + def __init__( + self, + target_head_pose: NDArray[np.float32], + start_head_pose: NDArray[np.float32] | None = None, + target_antennas: Tuple[float, float] = (0, 0), + start_antennas: Tuple[float, float] | None = None, + target_body_yaw: float = 0, + start_body_yaw: float | None = None, + duration: float = 1.0, + ): + """Initialize a GotoQueueMove.""" + self._duration = duration + self.target_head_pose = target_head_pose + self.start_head_pose = start_head_pose + self.target_antennas = target_antennas + self.start_antennas = start_antennas or (0, 0) + self.target_body_yaw = target_body_yaw + self.start_body_yaw = start_body_yaw or 0 + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return self._duration + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate goto move at time t using linear interpolation.""" + try: + from reachy_mini.utils import create_head_pose + from reachy_mini.utils.interpolation import linear_pose_interpolation + + # Clamp t to [0, 1] for interpolation + t_clamped = max(0, min(1, t / self.duration)) + + # Use start pose if available, otherwise neutral + if self.start_head_pose is not None: + start_pose = self.start_head_pose + else: + start_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + + # Interpolate head pose + head_pose = linear_pose_interpolation(start_pose, self.target_head_pose, t_clamped) + + # Interpolate antennas - return as numpy array + antennas = np.array( + [ + self.start_antennas[0] + (self.target_antennas[0] - self.start_antennas[0]) * t_clamped, + self.start_antennas[1] + (self.target_antennas[1] - self.start_antennas[1]) * t_clamped, + ], + dtype=np.float64, + ) + + # Interpolate body yaw + body_yaw = self.start_body_yaw + (self.target_body_yaw - self.start_body_yaw) * t_clamped + + return (head_pose, antennas, body_yaw) + + except Exception as e: + logger.error(f"Error evaluating goto move at t={t}: {e}") + # Return target pose on error - convert to float64 + target_head_pose_f64 = self.target_head_pose.astype(np.float64) + target_antennas_array = np.array([self.target_antennas[0], self.target_antennas[1]], dtype=np.float64) + return (target_head_pose_f64, target_antennas_array, self.target_body_yaw) diff --git a/src/peachy/huggingface_realtime.py b/src/peachy/huggingface_realtime.py new file mode 100644 index 0000000000000000000000000000000000000000..2fee7081dffafbf3af6c9573a5094a60ace4e9bc --- /dev/null +++ b/src/peachy/huggingface_realtime.py @@ -0,0 +1,1133 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +import json +import time +import uuid +import base64 +import random +import asyncio +import logging +from typing import Any, Final, Optional +from urllib.parse import urlsplit, urlunsplit + +import httpx +import numpy as np +from openai import AsyncOpenAI +from pydantic import Field, BaseModel +from typing_extensions import Literal, TypedDict +from openai.types.realtime import ( + AudioTranscriptionParam, + RealtimeAudioConfigParam, + RealtimeToolsConfigParam, + RealtimeFunctionToolParam, + RealtimeAudioConfigInputParam, + RealtimeAudioConfigOutputParam, + RealtimeSessionCreateRequestParam, +) +from websockets.exceptions import ConnectionClosedError +from openai.resources.realtime.realtime import AsyncRealtimeConnection +from openai.types.realtime.realtime_audio_input_turn_detection_param import ServerVad + +from peachy.tools import core_tools +from peachy.config import ( + HF_LOCAL_CONNECTION_MODE, + config, + get_default_voice, + get_available_voices, + get_hf_direct_ws_url, + parse_hf_realtime_url, + get_hf_connection_selection, +) +from peachy.prompts import ( + get_session_voice, + get_session_instructions, + get_session_greeting_prompt, +) +from peachy.streaming import AdditionalOutputs, audio_to_int16 +from peachy.tools.core_tools import ( + ToolSpec, + ToolDependencies, + get_tool_specs, +) +from peachy.conversation_handler import AudioFrame, QueueItem, ConversationHandler +from peachy.providers import redact_secrets +from peachy.tools.background_tool_manager import ( + ToolCallRoutine, + ToolNotification, + BackgroundToolManager, +) + + +logger = logging.getLogger(__name__) + +_RESPONSE_DONE_TIMEOUT: Final[float] = 30.0 +_RESPONSE_REJECTION_RETRY_DELAY: Final[float] = 0.5 + + +def _safe_endpoint_for_log(endpoint: str) -> str: + """Render an endpoint without userinfo, query parameters, or fragments.""" + parsed = urlsplit(endpoint) + host = parsed.hostname or "" + if ":" in host and not host.startswith("["): + host = f"[{host}]" + netloc = f"{host}:{parsed.port}" if parsed.port is not None else host + return urlunsplit((parsed.scheme, netloc, parsed.path, "", "")) + + +class InputTranscriptChunksByItem(BaseModel): + """Current item_id and its accumulated deltas. Only one item at a time.""" + + item_id: str | None = None + deltas: list[str] = Field(default_factory=list) + + +def to_realtime_tools_config(tool_specs: list[ToolSpec]) -> RealtimeToolsConfigParam: + """Convert app tool specs to the OpenAI-compatible realtime session shape.""" + realtime_tools: RealtimeToolsConfigParam = [] + for spec in tool_specs: + realtime_tools.append( + RealtimeFunctionToolParam( + type="function", + name=spec["name"], + description=spec["description"], + parameters=spec["parameters"], + ) + ) + return realtime_tools + + +class HFNativeRateAudioPCM(TypedDict): + """Hugging Face extension for native-rate PCM audio.""" + + type: Literal["audio/pcm"] + rate: None + + +def _native_rate_audio_pcm() -> HFNativeRateAudioPCM: + """Return the Hugging Face native-rate PCM config.""" + return {"type": "audio/pcm", "rate": None} + + +def _build_openai_compatible_client_from_realtime_url( + realtime_url: str, + bearer_token: str | None, +) -> tuple[AsyncOpenAI, dict[str, str]]: + """Build an OpenAI-compatible realtime client from a direct websocket/base URL.""" + parsed = parse_hf_realtime_url(realtime_url) + client = AsyncOpenAI( + api_key=bearer_token or "DUMMY", + base_url=parsed.base_url, + websocket_base_url=parsed.websocket_base_url, + ) + return client, parsed.connect_query + + +class HuggingFaceRealtimeHandler(ConversationHandler): + """Realtime stream handler for the Hugging Face OpenAI-compatible endpoint.""" + + SAMPLE_RATE = 16000 + + def __init__( + self, + deps: ToolDependencies, + instance_path: Optional[str] = None, + startup_voice: Optional[str] = None, + ): + """Initialize the handler.""" + super().__init__() + + self.deps = deps + + self.client: AsyncOpenAI + self.connection: AsyncRealtimeConnection | None = None + self.output_queue: asyncio.Queue[QueueItem] = asyncio.Queue() + + self.instance_path = instance_path + self._voice_override: str | None = self._normalize_startup_voice(startup_voice) + self._realtime_connect_query: dict[str, str] = {} + + # Debouncing for partial transcripts + self.partial_transcript_task: asyncio.Task[None] | None = None + self.partial_debounce_delay = 0.5 # seconds + self.input_transcript_chunks_by_item = InputTranscriptChunksByItem() + + # Internal lifecycle flags + self._connected_event: asyncio.Event = asyncio.Event() + + # Background tool manager + self.tool_manager = BackgroundToolManager() + + # Response-in-progress guard: the Realtime API only allows one active + # response per conversation at a time. A dedicated worker task + # (_response_sender_loop) dequeues and sends one request at a time + self._pending_responses: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + self._response_done_event: asyncio.Event = asyncio.Event() + self._response_done_event.set() + self._response_started_or_rejected_event: asyncio.Event = asyncio.Event() + self._last_response_rejected: bool = False + self._turn_user_done_at: float | None = None + self._turn_response_created_at: float | None = None + self._turn_first_audio_at: float | None = None + self._startup_greeting_sent = False + self._in_flight_tool_calls: set[str] = set() + self._tool_batch_needs_response = False + self._restart_tasks: set[asyncio.Task[None]] = set() + self._restart_lock = asyncio.Lock() + self._shutdown_started = False + + @staticmethod + def _sanitize_tool_result_for_model(tool_name: str, tool_result: dict[str, Any]) -> dict[str, Any]: + """Remove bulky transport-only fields before echoing tool output back to the model.""" + if tool_name == "camera" and "b64_im" in tool_result: + sanitized = dict(tool_result) + sanitized.pop("b64_im", None) + sanitized["image_attached"] = True + return sanitized + return tool_result + + def _normalize_startup_voice(self, voice: str | None) -> str | None: + """Return a valid persisted startup voice, or None.""" + return self._resolve_backend_voice(voice, source="persisted startup voice") + + def _restart_task_done(self, task: asyncio.Task[None]) -> None: + """Retrieve and sanitize background restart failures.""" + self._restart_tasks.discard(task) + if task.cancelled(): + return + try: + task.result() + except Exception as exc: + logger.warning("Realtime session restart failed: %s", redact_secrets(exc)) + + async def _cancel_restart_tasks(self) -> None: + """Cancel and retrieve retained restart tasks during handler shutdown.""" + restart_tasks = tuple(self._restart_tasks) + for task in restart_tasks: + task.cancel() + if restart_tasks: + await asyncio.gather(*restart_tasks, return_exceptions=True) + self._restart_tasks.clear() + + async def _wait_for_response_done_before_tool_result(self) -> bool: + """Return whether the function-call response finished before sending tool output.""" + if self._response_done_event.is_set(): + return True + + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + return True + except asyncio.TimeoutError: + return False + + def _resolve_backend_voice( + self, + voice: str | None, + *, + source: str, + fallback: str | None = None, + ) -> str | None: + """Return a backend-supported voice, optionally falling back when unsupported.""" + available_voices = get_available_voices() + voice_value = (voice or "").strip() + if not voice_value: + return fallback + + voice_by_lowercase = {candidate.lower(): candidate for candidate in available_voices} + normalized_voice = voice_by_lowercase.get(voice_value.lower()) + if normalized_voice is not None: + return normalized_voice + + if voice: + logger.warning( + "Ignoring unsupported %s %r; expected one of %s", + source, + voice, + available_voices, + ) + return fallback + + def _get_session_config(self, tool_specs: list[ToolSpec]) -> RealtimeSessionCreateRequestParam: + """Return the Hugging Face OpenAI-compatible session config.""" + return RealtimeSessionCreateRequestParam( + type="realtime", + instructions=get_session_instructions(self.instance_path), + audio=RealtimeAudioConfigParam( + input=RealtimeAudioConfigInputParam( + # The OpenAI SDK type only includes 24 kHz PCM, but the HF + # compatible server uses rate=None for native 16 kHz mode. + format=_native_rate_audio_pcm(), # type: ignore[typeddict-item] + transcription=AudioTranscriptionParam( + model="gpt-4o-transcribe", + language=config.REALTIME_TRANSCRIPTION_LANGUAGE, + ), + turn_detection=ServerVad(type="server_vad", interrupt_response=True), + ), + output=RealtimeAudioConfigOutputParam( + format=_native_rate_audio_pcm(), # type: ignore[typeddict-item] + voice=self.get_current_voice(), + ), + ), + tools=to_realtime_tools_config(tool_specs), + tool_choice="auto", + ) + + def _is_connected(self) -> bool: + """Return whether the realtime connection is open.""" + return self.connection is not None + + def _idle_behavior_ready(self) -> bool: + """Hold idle behavior while a model response is still active.""" + return self._response_done_event.is_set() + + async def _cancel_partial_transcript_task(self) -> None: + if self.partial_transcript_task and not self.partial_transcript_task.done(): + self.partial_transcript_task.cancel() + try: + await self.partial_transcript_task + except asyncio.CancelledError: + pass + + async def change_voice(self, voice: str) -> str: + """Change only the voice, updating the active session when possible.""" + default_voice = get_default_voice() + resolved_voice = ( + self._resolve_backend_voice(voice, source="requested voice", fallback=default_voice) or default_voice + ) + self._voice_override = resolved_voice + if self.connection is not None: + try: + await self.connection.session.update( + session=RealtimeSessionCreateRequestParam( + type="realtime", + audio=RealtimeAudioConfigParam( + output=RealtimeAudioConfigOutputParam( + voice=resolved_voice, + ), + ), + ), + ) + return f"Voice changed to {resolved_voice}." + except Exception as e: + logger.warning("Failed to update live session for voice change: %s", redact_secrets(e)) + return "Voice change failed. Will take effect on next connection." + return "Voice changed. Will take effect on next connection." + + def get_current_voice(self) -> str: + """Return the voice currently selected for this handler.""" + default_voice = get_default_voice() + voice = self._voice_override or get_session_voice(default=default_voice) + return self._resolve_backend_voice(voice, source="session voice", fallback=default_voice) or default_voice + + async def apply_personality(self, profile: str | None) -> str: + """Apply a new personality (profile) at runtime if possible. + + - Updates the global config's selected profile for subsequent calls. + - If a realtime connection is active, sends a session.update with the + freshly resolved instructions so the change takes effect immediately. + + Returns a short status message for UI feedback. + """ + try: + # Update the in-process config value and env + from peachy.config import config as _config + from peachy.config import set_custom_profile + + set_custom_profile(profile) + logger.info( + "Set custom profile to %r (config=%r)", profile, getattr(_config, "REACHY_MINI_CUSTOM_PROFILE", None) + ) + + try: + instructions = get_session_instructions(self.instance_path) + voice = self.get_current_voice() + except Exception as e: + safe_error = redact_secrets(e) + logger.error("Failed to resolve personality content: %s", safe_error) + return f"Failed to apply personality: {safe_error}" + + # Rebuild the tool registry + core_tools.initialize_tools(force=True) + + # Attempt a live update first, then force a full restart to ensure it sticks + if self.connection is not None: + try: + await self.connection.session.update( + session=RealtimeSessionCreateRequestParam( + type="realtime", + instructions=instructions, + audio=RealtimeAudioConfigParam( + output=RealtimeAudioConfigOutputParam( + voice=voice, + ), + ), + ), + ) + logger.info("Applied personality via live update: %s", profile or "built-in default") + except Exception as e: + logger.warning("Live update failed; will restart session: %s", redact_secrets(e)) + + # Force a real restart to guarantee the new instructions/voice + try: + await self._restart_session() + return "Applied personality and restarted realtime session." + except Exception as e: + logger.warning("Failed to restart session after apply: %s", redact_secrets(e)) + return "Applied personality. Will take effect on next connection." + else: + logger.info( + "Applied personality recorded: %s (no live connection; will apply on next session)", + profile or "built-in default", + ) + return "Applied personality. Will take effect on next connection." + except Exception as e: + safe_error = redact_secrets(e) + logger.error("Error applying personality '%s': %s", profile, safe_error) + return f"Failed to apply personality: {safe_error}" + + async def _emit_debounced_partial(self, transcript: str, item_id: str, sequence_counter: int) -> None: + """Emit partial transcript after debounce delay.""" + try: + await asyncio.sleep(self.partial_debounce_delay) + + input_transcript = self.input_transcript_chunks_by_item + if input_transcript.item_id == item_id and len(input_transcript.deltas) - 1 == sequence_counter: + await self.output_queue.put(AdditionalOutputs({"role": "user_partial", "content": transcript})) + logger.debug(f"Debounced partial emitted: {transcript}") + except asyncio.CancelledError: + logger.debug("Debounced partial cancelled") + raise + + def _record_partial_transcript_delta( + self, + input_transcript: InputTranscriptChunksByItem, + item_id: str, + delta: str, + ) -> None: + """Record a Hugging Face partial transcript snapshot.""" + input_transcript.item_id = item_id + input_transcript.deltas = [delta] + + async def start_up(self) -> None: + """Start the handler with minimal retries on unexpected websocket closure.""" + self.client = await self._build_realtime_client() + + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + await self._run_realtime_session() + # Normal exit from the session, stop retrying + return + except ConnectionClosedError as e: + # Abrupt close (e.g., "no close frame received or sent") → retry + logger.warning( + "Realtime websocket closed unexpectedly (attempt %d/%d): %s", + attempt, + max_attempts, + redact_secrets(e), + ) + if attempt < max_attempts: + self.client = await self._build_realtime_client() + # exponential backoff with jitter + base_delay = 2 ** (attempt - 1) # 1s, 2s, 4s, 8s, etc. + jitter = random.uniform(0, 0.5) + delay = base_delay + jitter + logger.info("Retrying in %.1f seconds...", delay) + await asyncio.sleep(delay) + continue + raise + finally: + # never keep a stale reference + self.connection = None + try: + self._connected_event.clear() + except Exception: + pass + + async def _restart_session(self) -> None: + """Force-close the current session and start a fresh one in background. + + Does not block the caller while the new session is establishing. + """ + try: + async with self._restart_lock: + if self._shutdown_started: + return + + if self.connection is not None: + try: + await self.connection.close() + except Exception: + pass + finally: + self.connection = None + + # Ensure we have a client (start_up must have run once) + if getattr(self, "client", None) is None: + logger.warning("Cannot restart: realtime client not initialized yet.") + return + + # Build before registering the session task, then re-check the + # shutdown gate because client construction may have suspended. + try: + self._connected_event.clear() + except Exception: + pass + client = await self._build_realtime_client() + if self._shutdown_started: + return + self.client = client + restart_task = asyncio.create_task( + self._run_realtime_session(), + name="realtime-session-restart", + ) + self._restart_tasks.add(restart_task) + restart_task.add_done_callback(self._restart_task_done) + try: + await asyncio.wait_for(self._connected_event.wait(), timeout=5.0) + if not self._shutdown_started: + logger.info("Realtime session restarted and connected.") + except asyncio.TimeoutError: + if not self._shutdown_started: + logger.warning("Realtime session restart timed out; continuing in background.") + except asyncio.CancelledError: + raise + except Exception as e: + logger.warning("_restart_session failed: %s", redact_secrets(e)) + + async def _safe_response_create(self, **kwargs: Any) -> None: + """Enqueue a response.create() kwargs for the sender worker _response_sender_loop(). + + This method never blocks the caller. + """ + await self._pending_responses.put(kwargs) + + async def _send_startup_greeting_prompt(self) -> None: + """Prompt the model to open the conversation once the session is ready.""" + if self._startup_greeting_sent or not self.connection: + return + + greeting_prompt = get_session_greeting_prompt().strip() + if not greeting_prompt: + self._startup_greeting_sent = True + return + + try: + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": greeting_prompt, + }, + ], + }, + ) + self._startup_greeting_sent = True + self._mark_activity("startup_greeting_prompt") + await self._safe_response_create() + logger.info("Queued startup greeting prompt") + except Exception as e: + logger.warning("Failed to queue startup greeting prompt: %s", redact_secrets(e)) + + async def _response_sender_loop(self) -> None: + """Dedicated worker that sends ``response.create()`` calls serially. + + This logic was designed to comply with the response.create() docstring specification for event ordering: + https://github.com/openai/openai-python/blob/3e0c05b84a2056870abf3bd6a5e7849020209cc3/src/openai/resources/realtime/realtime.py#L649C1-L651C30 + + For each queued request the worker: + 1. Waits until no response is active (_response_done_event). + 2. Sends response.create(). + 3. Waits until the receiver observes response.created or a rejection. + 4. Waits for the response cycle to complete (response.done). + 5. If the server rejected with active_response, retries from step 1. + """ + while self.connection: + try: + kwargs = await self._pending_responses.get() + except asyncio.CancelledError: + return + + # Parallel tool calls enqueue duplicate empty requests; coalesce to one. + while not kwargs and not self._pending_responses.empty(): + try: + self._pending_responses.get_nowait() + except asyncio.QueueEmpty: + break + + sent = False + max_retries = 5 + attempts = 0 + while not sent and self.connection and attempts < max_retries: + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for previous response to finish; forcing ahead") + self._response_done_event.set() + + if not self.connection: + break + + self._last_response_rejected = False + self._response_started_or_rejected_event.clear() + try: + await self.connection.response.create(**kwargs) + except Exception as e: + logger.debug("_response_sender_loop: send failed: %s", redact_secrets(e)) + self._response_done_event.set() + break + + try: + await asyncio.wait_for( + self._response_started_or_rejected_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for response.created or response rejection") + + # Check if the receiver loop observed an asynchronous rejection. + if self._last_response_rejected: + attempts += 1 + if attempts >= max_retries: + logger.debug("response.create rejected %d times; giving up", attempts) + break + logger.debug("response.create was rejected; retrying (%d/%d)", attempts, max_retries) + await asyncio.sleep(_RESPONSE_REJECTION_RETRY_DELAY) + continue + + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for response.done; assuming response completed") + self._response_done_event.set() + break + + sent = True + + async def _handle_tool_result(self, completed_tool: ToolNotification) -> None: + """Process the result of a tool call.""" + if completed_tool.error is not None: + logger.error( + "Tool '%s' (id=%s) failed with error: %s", + completed_tool.tool_name, + completed_tool.id, + completed_tool.error, + ) + tool_result = {"error": completed_tool.error} + tool_result_for_model = tool_result + elif completed_tool.result is not None: + tool_result = completed_tool.result + tool_result_for_model = ( + self._sanitize_tool_result_for_model(completed_tool.tool_name, tool_result) + if isinstance(tool_result, dict) + else tool_result + ) + logger.info( + "Tool '%s' (id=%s) executed successfully.", + completed_tool.tool_name, + completed_tool.id, + ) + logger.debug("Tool '%s' model-visible result: %s", completed_tool.tool_name, tool_result_for_model) + else: + logger.warning( + "Tool '%s' (id=%s) returned no result and no error", completed_tool.tool_name, completed_tool.id + ) + tool_result = {"error": "No result returned from tool execution"} + tool_result_for_model = tool_result + + # Connection may have closed while tool was running + if not self.connection: + logger.warning( + "Connection closed during tool '%s' (id=%s) execution; cannot send result back", + completed_tool.tool_name, + completed_tool.id, + ) + return + + try: + send_result_to_model = not completed_tool.is_idle_tool_call + if send_result_to_model: + self._mark_activity("tool_result_ready") + model_result_submitted = False + if send_result_to_model and isinstance(completed_tool.id, str): + if not await self._wait_for_response_done_before_tool_result(): + send_result_to_model = False + if not send_result_to_model: + logger.warning( + "Dropping realtime model result for tool '%s' (id=%s) because response.done was not observed", + completed_tool.tool_name, + completed_tool.id, + ) + elif not self.connection: + logger.warning( + "Connection closed before sending tool '%s' (id=%s) result back", + completed_tool.tool_name, + completed_tool.id, + ) + return + else: + await self.connection.conversation.item.create( + item={ + "type": "function_call_output", + "call_id": completed_tool.id, + "output": json.dumps(tool_result_for_model), + }, + ) + model_result_submitted = True + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": json.dumps(tool_result_for_model), + }, + ), + ) + + if model_result_submitted and completed_tool.tool_name == "camera" and "b64_im" in tool_result: + # use raw base64, don't json.dumps (which adds quotes) + b64_im = tool_result["b64_im"] + if not isinstance(b64_im, str): + logger.warning("Unexpected type for b64_im: %s", type(b64_im)) + b64_im = str(b64_im) + image_width = tool_result.get("image_width") + image_height = tool_result.get("image_height") + jpeg_bytes_value = tool_result.get("jpeg_bytes") + jpeg_bytes = jpeg_bytes_value if isinstance(jpeg_bytes_value, int) else (len(b64_im) * 3) // 4 + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": f"data:image/jpeg;base64,{b64_im}", + }, + ], + }, + ) + if isinstance(image_width, int) and isinstance(image_height, int): + logger.info( + "Added camera image to conversation frame=%sx%s jpeg_bytes=%s", + image_width, + image_height, + jpeg_bytes, + ) + else: + logger.info( + "Added camera image to conversation jpeg_bytes=%s", + jpeg_bytes, + ) + + if isinstance(completed_tool.id, str): + self._in_flight_tool_calls.discard(completed_tool.id) + + tool = core_tools.ALL_TOOLS.get(completed_tool.tool_name) + # Always surface errors, skip the spoken follow-up for tools that opt out. + if model_result_submitted and (completed_tool.error is not None or tool is None or tool.needs_response): + self._tool_batch_needs_response = True + + # Parallel tool calls in one turn: respond once every result is in, not per tool. + if self._tool_batch_needs_response and not self._in_flight_tool_calls: + self._tool_batch_needs_response = False + await self._safe_response_create() + + except ConnectionClosedError: + logger.warning("Connection closed while sending tool result") + self.connection = None + self._response_done_event.set() + + async def _run_realtime_session(self) -> None: + """Establish and manage a single realtime session.""" + tool_specs = get_tool_specs() + logger.info( + "Tools to be used in conversation: %s", + [tool["name"] for tool in tool_specs], + ) + connect_kwargs: dict[str, Any] = {} + if self._realtime_connect_query: + connect_kwargs["extra_query"] = self._realtime_connect_query + async with self.client.realtime.connect(**connect_kwargs) as conn: + try: + session_config = self._get_session_config(tool_specs) + await conn.session.update(session=session_config) + logger.info( + "Realtime session initialized with profile=%r voice=%r", + getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None), + self.get_current_voice(), + ) + except Exception as exc: + logger.error( + "Realtime session.update failed; aborting startup: %s", + redact_secrets(exc), + ) + raise + + logger.info("Realtime session updated successfully") + + # Reset the partial-transcript accumulator for each new session + self.input_transcript_chunks_by_item = InputTranscriptChunksByItem() + + # Manage events received from the realtime server. + self.connection = conn + try: + self._connected_event.set() + except Exception: + pass + + response_sender_task: asyncio.Task[None] | None = None + try: + # Start the background tool manager + self.tool_manager.start_up(tool_callbacks=[self._handle_tool_result]) + + # Start the response sender worker + response_sender_task = asyncio.create_task(self._response_sender_loop(), name="response-sender") + await self._send_startup_greeting_prompt() + + async for event in self.connection: + logger.debug("Realtime event: %s", event.type) + if event.type == "input_audio_buffer.speech_started": + self._mark_activity("user_speech_started") + self._turn_user_done_at = None + self._turn_response_created_at = None + self._turn_first_audio_at = None + if self._clear_queue: + self._clear_queue() + self.deps.movement_manager.set_listening(True) + logger.debug("User speech started") + + if event.type == "input_audio_buffer.speech_stopped": + self._mark_activity("user_speech_stopped") + self.deps.movement_manager.set_listening(False) + logger.debug("User speech stopped - server will auto-commit with VAD") + + if event.type == "response.output_audio.done": + self.deps.movement_manager.set_speaking(False) + logger.debug("response completed") + + if event.type == "response.output_text.delta": + logger.debug("response text delta") + + if event.type == "response.output_text.done": + logger.debug("response text done: %s", event.text) + + if event.type == "response.created": + self._mark_activity("response_created") + self.deps.movement_manager.set_speaking(True) + self._response_done_event.clear() + self._response_started_or_rejected_event.set() + if self._turn_user_done_at is not None and self._turn_response_created_at is None: + self._turn_response_created_at = time.perf_counter() + delta_ms = (self._turn_response_created_at - self._turn_user_done_at) * 1000 + logger.info("Turn latency: response.created %.0f ms after user transcript", delta_ms) + logger.debug("Response created (active)") + + if event.type == "response.done": + # Doesn't mean the audio is done playing + # Resume tracking for responses that emit no audio (text-only / tool-only). + self.deps.movement_manager.set_speaking(False) + self._response_done_event.set() + self._response_started_or_rejected_event.set() + logger.debug("Response done") + + if event.type == "conversation.item.input_audio_transcription.delta": + self._mark_activity("user_transcription_delta") + logger.debug(f"User partial transcript: {event.delta}") + + item_id = event.item_id + delta = event.delta or "" + + input_transcript = self.input_transcript_chunks_by_item + self._record_partial_transcript_delta(input_transcript, item_id, delta) + + current_partial = "".join(input_transcript.deltas) + sequence_counter = len(input_transcript.deltas) - 1 + + await self._cancel_partial_transcript_task() + + # Start new debounce timer with the last delta + self.partial_transcript_task = asyncio.create_task( + self._emit_debounced_partial(current_partial, item_id, sequence_counter) + ) + + # Handle completed transcription (user finished speaking) + if event.type == "conversation.item.input_audio_transcription.completed": + self._mark_activity("user_transcription_completed") + raw_transcript = event.transcript or "" + transcript = raw_transcript.strip() + logger.debug("User transcript: %s", raw_transcript) + self.deps.movement_manager.set_listening(False) + + await self._cancel_partial_transcript_task() + + if not transcript: + logger.debug("Ignoring empty user transcript") + continue + + self._turn_user_done_at = time.perf_counter() + self._turn_response_created_at = None + self._turn_first_audio_at = None + self._in_flight_tool_calls.clear() + self._tool_batch_needs_response = False + + await self.output_queue.put(AdditionalOutputs({"role": "user", "content": transcript})) + + # Handle assistant transcription + if event.type == "response.output_audio_transcript.done": + self._mark_activity("assistant_transcript_done") + logger.debug(f"Assistant transcript: {event.transcript}") + await self.output_queue.put( + AdditionalOutputs({"role": "assistant", "content": event.transcript}) + ) + + # Handle audio delta + if event.type == "response.output_audio.delta": + decoded_pcm_bytes = base64.b64decode(event.delta) + decoded_pcm = np.frombuffer(decoded_pcm_bytes, dtype=np.int16).reshape(1, -1) + self._mark_activity("assistant_audio_delta") + if self._turn_user_done_at is not None and self._turn_first_audio_at is None: + self._turn_first_audio_at = time.perf_counter() + delta_ms = (self._turn_first_audio_at - self._turn_user_done_at) * 1000 + logger.info("Turn latency: first audio delta %.0f ms after user transcript", delta_ms) + await self.output_queue.put( + ( + self.SAMPLE_RATE, + decoded_pcm, + ), + ) + # ---- tool-calling plumbing ---- + if event.type == "response.function_call_arguments.done": + self._mark_activity("tool_call_received") + tool_name = getattr(event, "name", None) + args_json_str = getattr(event, "arguments", None) + call_id: str = str(getattr(event, "call_id", uuid.uuid4())) + + logger.info( + "Tool call received — tool_name=%r, call_id=%s, args=%s", + tool_name, + call_id, + args_json_str, + ) + + if not isinstance(tool_name, str) or not isinstance(args_json_str, str): + logger.error( + "Invalid tool call: tool_name=%s (type=%s), args=%s (type=%s), call_id=%s", + tool_name, + type(tool_name).__name__, + args_json_str, + type(args_json_str).__name__, + call_id, + ) + continue + + self._in_flight_tool_calls.add(call_id) + background_tool = await self.tool_manager.start_tool( + call_id=call_id, + tool_call_routine=ToolCallRoutine( + tool_name=tool_name, + args_json_str=args_json_str, + deps=self.deps, + ), + is_idle_tool_call=False, + ) + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": f"🛠️ Used tool {tool_name} with args {args_json_str}. The tool is now running. Tool ID: {background_tool.tool_id}", + }, + ), + ) + logger.info( + "Started background tool: %s (id=%s, call_id=%s)", + tool_name, + background_tool.tool_id, + call_id, + ) + + # server error + if event.type == "error": + err = getattr(event, "error", None) + msg = getattr(err, "message", str(err) if err else "unknown error") + code = getattr(err, "code", "") or getattr(err, "type", "") + + if code == "conversation_already_has_active_response": + # response.create was rejected. The sender worker + # is waiting on _response_done_event; when the active + # response finishes it will wake up and see this flag. + self._last_response_rejected = True + self._response_started_or_rejected_event.set() + logger.debug("response.create rejected; worker will retry after active response finishes") + else: + self._response_started_or_rejected_event.set() + logger.error( + "Realtime error [%s]: %s (raw=%s)", + code, + redact_secrets(msg), + redact_secrets(err), + ) + + if code == "input_audio_buffer_commit_empty": + self.deps.movement_manager.set_listening(False) + + # Only show user-facing errors, not internal state errors. + if code not in ( + "input_audio_buffer_commit_empty", + "conversation_already_has_active_response", + ): + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": f"[error] {redact_secrets(msg)}", + } + ) + ) + finally: + # Stop the response sender worker. + if response_sender_task is not None: + response_sender_task.cancel() + try: + await response_sender_task + except asyncio.CancelledError: + pass + + # Stop background tool manager tasks (listener + cleanup) in all paths. + await self.tool_manager.shutdown() + + # Microphone receive + async def receive(self, frame: AudioFrame) -> None: + """Receive audio frame from the microphone and send it to the realtime server. + + Handles both mono and stereo audio formats, converting to the expected + mono format for the realtime API. + + Args: + frame: A tuple containing (sample_rate, audio_data). + + """ + if not self.connection: + return + + _, audio_frame = frame + if audio_frame.size == 0: + return + + # Reshape if needed + if audio_frame.ndim == 2: + # channels-last convention + if audio_frame.shape[1] > audio_frame.shape[0]: + audio_frame = audio_frame.T + # Multiple channels -> Mono channel + if audio_frame.shape[1] > 1: + audio_frame = audio_frame[:, 0] + + # Cast if needed + audio_frame = audio_to_int16(audio_frame) + + # Send to the realtime input buffer (guard against races during reconnect). + try: + audio_message = base64.b64encode(audio_frame.tobytes()).decode("utf-8") + await self.connection.input_audio_buffer.append(audio=audio_message) + except Exception as e: + logger.debug("Dropping audio frame: connection not ready (%s)", redact_secrets(e)) + return + + async def shutdown(self) -> None: + """Shutdown the handler.""" + self._shutdown_started = True + self._connected_event.set() + + # Wait for any restart still constructing its client to observe the + # shutdown gate before taking the complete registered-task snapshot. + async with self._restart_lock: + pass + + # Unblock the response sender worker so it can exit + self._response_done_event.set() + + await self._cancel_restart_tasks() + + # Stop background tool manager tasks (listener + cleanup) + await self.tool_manager.shutdown() + + await self._cancel_partial_transcript_task() + + if self.connection: + try: + await self.connection.close() + except ConnectionClosedError as e: + logger.debug("Connection already closed during shutdown: %s", redact_secrets(e)) + except Exception as e: + logger.debug("connection.close() ignored: %s", redact_secrets(e)) + finally: + self.connection = None + + # Clear any remaining items in the output queue + while not self.output_queue.empty(): + try: + self.output_queue.get_nowait() + except asyncio.QueueEmpty: + break + + async def get_available_voices(self) -> list[str]: + """Return the available Hugging Face voices.""" + return get_available_voices() + + async def _build_realtime_client(self) -> AsyncOpenAI: + """Build the Hugging Face OpenAI-compatible realtime client.""" + bearer_token = (config.HF_TOKEN or "").strip() + connection_selection = get_hf_connection_selection() + direct_realtime_url = get_hf_direct_ws_url() + if connection_selection.mode == HF_LOCAL_CONNECTION_MODE: + if not direct_realtime_url: + raise RuntimeError("HF_REALTIME_WS_URL must be set when HF_REALTIME_CONNECTION_MODE=local") + client, connect_query = _build_openai_compatible_client_from_realtime_url( + direct_realtime_url, + bearer_token, + ) + self._realtime_connect_query = connect_query + logger.info( + "Using direct Hugging Face realtime endpoint %s", + redact_secrets(_safe_endpoint_for_log(direct_realtime_url)), + ) + return client + + session_url = connection_selection.session_url + if not session_url: + raise RuntimeError("Built-in Hugging Face session proxy URL is unavailable") + if direct_realtime_url: + logger.info("HF_REALTIME_CONNECTION_MODE=deployed; ignoring HF_REALTIME_WS_URL.") + + allocator_headers = {"Authorization": f"Bearer {bearer_token}"} if bearer_token else None + async with httpx.AsyncClient(timeout=10.0) as http_client: + response = await http_client.post(session_url, headers=allocator_headers) + response.raise_for_status() + payload = response.json() + + connect_url = payload.get("connect_url") + if not isinstance(connect_url, str) or not connect_url: + raise RuntimeError(f"Session allocator response did not contain a valid connect_url: {payload!r}") + + parsed_connect_url = parse_hf_realtime_url(connect_url) + if not parsed_connect_url.has_realtime_path: + raise ValueError(f"Expected realtime connect URL ending with /realtime, got: {connect_url}") + + logger.info("Allocated realtime session %s", payload.get("session_id") or "") + client, connect_query = _build_openai_compatible_client_from_realtime_url( + connect_url, + bearer_token, + ) + self._realtime_connect_query = connect_query + return client diff --git a/src/peachy/idle_policy.py b/src/peachy/idle_policy.py new file mode 100644 index 0000000000000000000000000000000000000000..cede9cc28a64e73fc9eda641490be9e2ccf1d4f7 --- /dev/null +++ b/src/peachy/idle_policy.py @@ -0,0 +1,140 @@ +"""Local idle tool selection and dispatch. + +Default idle weights among active, registered tools: +- IdleDoNothing: 60% +- Dance: 16% +- PlayEmotion: 16% +- MoveHead: 8% + +Profiles or sessions can disable tools. When one or more idle tools are unavailable, selection is renormalized across the +remaining active candidates. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import json +import uuid +import random +import asyncio +import logging +from typing import Any, Final +from dataclasses import dataclass +from collections.abc import Mapping, Callable, Iterable + +from peachy.tools import core_tools +from peachy.streaming import AdditionalOutputs +from peachy.tools.dance import Dance +from peachy.tools.move_head import MoveHead +from peachy.tools.play_emotion import PlayEmotion +from peachy.tools.idle_do_nothing import IdleDoNothing +from peachy.tools.background_tool_manager import ( + BackgroundTool, + ToolCallRoutine, + BackgroundToolManager, +) + + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class IdleToolCandidate: + """Weighted idle tool candidate and argument factory.""" + + tool_type: type[core_tools.Tool] + weight: float + args_factory: Callable[[], dict[str, Any]] + + +def _no_args() -> dict[str, Any]: + return {} + + +def _idle_do_nothing_args() -> dict[str, Any]: + return {"reason": "random idle policy selected stillness"} + + +def _move_head_args() -> dict[str, Any]: + return {"direction": random.choice(tuple(MoveHead.DELTAS))} + + +_IDLE_TOOL_CANDIDATES: Final[tuple[IdleToolCandidate, ...]] = ( + IdleToolCandidate(IdleDoNothing, 0.60, _idle_do_nothing_args), + IdleToolCandidate(Dance, 0.16, _no_args), + IdleToolCandidate(PlayEmotion, 0.16, _no_args), + IdleToolCandidate(MoveHead, 0.08, _move_head_args), +) + + +def choose_idle_tool_call( + available_tool_names: Iterable[str], + *, + tool_registry: Mapping[str, core_tools.Tool] | None = None, +) -> tuple[str, dict[str, Any]] | None: + """Choose a weighted idle tool call from the tools available to the session.""" + available = set(available_tool_names) + registry = core_tools.ALL_TOOLS if tool_registry is None else tool_registry + candidates = [ + (tool.name, candidate) + for candidate in _IDLE_TOOL_CANDIDATES + for tool in registry.values() + if tool.name in available and isinstance(tool, candidate.tool_type) + ] + if not candidates: + return None + + selected_name, selected_candidate = random.choices( + candidates, + weights=[candidate.weight for _, candidate in candidates], + k=1, + )[0] + return selected_name, selected_candidate.args_factory() + + +async def start_idle_tool_call( + *, + deps: core_tools.ToolDependencies, + tool_manager: BackgroundToolManager, + output_queue: asyncio.Queue[Any], + available_tool_names: Iterable[str], + idle_duration: float, +) -> BackgroundTool | None: + """Start a locally selected idle tool and publish the console notification.""" + selected_tool = choose_idle_tool_call(available_tool_names) + if selected_tool is None: + logger.warning("No idle tools are available; idle action skipped") + return None + + tool_name, arguments = selected_tool + args_json_str = json.dumps(arguments) + call_id = f"idle-{uuid.uuid4()}" + background_tool = await tool_manager.start_tool( + call_id=call_id, + tool_call_routine=ToolCallRoutine( + tool_name=tool_name, + args_json_str=args_json_str, + deps=deps, + ), + is_idle_tool_call=True, + ) + await output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": ( + f"🛠️ Idle tool {tool_name} with args {args_json_str}. " + f"The tool is now running. Tool ID: {background_tool.tool_id}" + ), + }, + ), + ) + logger.info( + "Started local idle tool after %.1fs idle: %s (id=%s, call_id=%s, args=%s)", + idle_duration, + tool_name, + background_tool.tool_id, + call_id, + args_json_str, + ) + return background_tool diff --git a/src/peachy/main.py b/src/peachy/main.py new file mode 100644 index 0000000000000000000000000000000000000000..a7acedc4d19dbc32da0c3ab1c5f652c38d9a2be0 --- /dev/null +++ b/src/peachy/main.py @@ -0,0 +1,372 @@ +"""Entrypoint for the Reachy Mini conversation app.""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import sys +import time +import asyncio +import logging +import argparse +import threading +from typing import TYPE_CHECKING, Any, Optional +from pathlib import Path +from collections.abc import Callable, Awaitable + +from fastapi import FastAPI, Request, Response + +from reachy_mini import ReachyMini, ReachyMiniApp +from peachy import app_lifecycle +from peachy.state import resolve_state_root +from peachy.utils import ( + parse_args, + setup_logger, + log_connection_troubleshooting, +) + + +if TYPE_CHECKING: + from peachy.console import LocalStream + from peachy.conversation_handler import ConversationHandler + from peachy.providers import ProviderDefinition + from peachy.tools.core_tools import ToolDependencies + + +def _build_selected_handler( + deps: "ToolDependencies", + instance_path: str | None, + startup_voice: str | None, +) -> tuple["ProviderDefinition", "ConversationHandler"]: + """Build the handler selected by the explicit provider registry.""" + from peachy.providers import get_provider_definition + + definition = get_provider_definition() + return definition, definition.build_handler(deps, instance_path, startup_voice) + + +def _start_inactivity_timeout_thread( + timeout_minutes: float, + stream_manager: LocalStream, + logger: logging.Logger, + app_stop_event: threading.Event | None, + go_to_sleep: Callable[[], dict[str, Any]] | None = None, +) -> threading.Thread: + """Start a daemon that puts the app to sleep after inactivity.""" + timeout_seconds = timeout_minutes * 60.0 + + def poll_inactivity_timeout() -> None: + logger.info("App inactivity timeout enabled: %.1f minutes.", timeout_minutes) + while app_stop_event is None or not app_stop_event.is_set(): + elapsed = stream_manager.seconds_since_activity() + if elapsed >= timeout_seconds: + logger.info("No activity for %.1f minutes; going to sleep.", elapsed / 60.0) + try: + if go_to_sleep is not None: + go_to_sleep() + else: + stream_manager.close() + except Exception as e: + logger.error("Error while going to sleep after inactivity timeout: %s", e) + try: + stream_manager.close() + except Exception as close_error: + logger.error("Error while closing stream manager after inactivity timeout: %s", close_error) + return + time.sleep(1.0) + + thread = threading.Thread(target=poll_inactivity_timeout, daemon=True) + thread.start() + return thread + + +def main() -> None: + """Entrypoint for the Reachy Mini conversation app.""" + args, _ = parse_args() + state_root = resolve_state_root() + if args.command == "tool-spaces": + from peachy.tool_spaces import handle_tool_spaces_command + + logger = setup_logger(args.debug) + try: + raise SystemExit(handle_tool_spaces_command(args, instance_path=state_root)) + except Exception as exc: + logger.error("tool-spaces command failed: %s", exc) + raise SystemExit(1) from exc + run(args, instance_path=str(state_root)) + + +def run( + args: argparse.Namespace, + robot: ReachyMini = None, + app_stop_event: Optional[threading.Event] = None, + settings_app: Optional[FastAPI] = None, + instance_path: Optional[str] = None, +) -> None: + """Run the Reachy Mini conversation app.""" + # Putting these dependencies here makes the dashboard faster to load when the conversation app is installed + from peachy.moves import MovementManager + from peachy.config import ( + set_instance_path, + resolve_app_timeout_minutes, + refresh_runtime_config_from_env, + ) + from peachy.startup_settings import ( + StartupSettings, + load_startup_settings_into_runtime, + ) + + logger = setup_logger(args.debug) + logger.info("Starting Reachy Mini Conversation App") + set_instance_path(instance_path) + startup_settings = StartupSettings() + + if instance_path is not None: + try: + from dotenv import load_dotenv + + env_path = Path(instance_path) / ".env" + if env_path.exists(): + load_dotenv(dotenv_path=str(env_path), override=True) + refresh_runtime_config_from_env() + logger.info("Loaded instance configuration from %s", env_path) + except Exception as e: + logger.warning("Failed to load instance configuration: %s", e) + + try: + startup_settings = load_startup_settings_into_runtime(instance_path) + except Exception as e: + logger.warning("Failed to load startup settings: %s", e) + + from peachy.console import LocalStream + from peachy.providers import get_provider_definition + from peachy.tools.core_tools import ToolDependencies, initialize_tools + + if robot is None: + try: + robot_kwargs = {} + if args.robot_name is not None: + robot_kwargs["robot_name"] = args.robot_name + + logger.info("Initializing ReachyMini (SDK will auto-detect appropriate backend)") + robot = ReachyMini(**robot_kwargs) + + except TimeoutError as e: + logger.error(f"Connection timeout: Failed to connect to Reachy Mini daemon. Details: {e}") + log_connection_troubleshooting(logger, args.robot_name) + sys.exit(1) + + except ConnectionError as e: + logger.error(f"Connection failed: Unable to establish connection to Reachy Mini. Details: {e}") + log_connection_troubleshooting(logger, args.robot_name) + sys.exit(1) + + except Exception as e: + logger.error(f"Unexpected error during robot initialization: {type(e).__name__}: {e}") + logger.error("Please check your configuration and try again.") + sys.exit(1) + + app_lifecycle.wake_up_if_sleeping(robot, logger) + + movement_manager = MovementManager(current_robot=robot) + + deps = ToolDependencies( + reachy_mini=robot, + movement_manager=movement_manager, + instance_path=instance_path, + camera_enabled=not args.no_camera, + ) + + def current_provider() -> ProviderDefinition: + return get_provider_definition() + + def build_handler(startup_voice: Optional[str] = None) -> ConversationHandler: + """Build the currently selected realtime handler.""" + definition, handler = _build_selected_handler(deps, instance_path, startup_voice) + logger.info("Selected realtime provider: %s", definition.name) + return handler + + handler = build_handler(startup_settings.voice) + + stream_manager: LocalStream | None = None + own_ui_server = None + + effective_settings_app = settings_app + if args.ui and settings_app is None: + effective_settings_app = FastAPI() + + @effective_settings_app.middleware("http") + async def _no_cache(request: Request, call_next: Callable[[Request], Awaitable[Response]]) -> Response: + """Serve everything no-store so browsers don't keep stale UI modules.""" + response = await call_next(request) + response.headers["Cache-Control"] = "no-store" + return response + + stream_manager = LocalStream( + handler, + robot, + settings_app=effective_settings_app, + instance_path=instance_path, + handler_factory=build_handler, + provider_resolver=current_provider, + startup_voice=startup_settings.voice, + ) + + # The page is served immediately, so the API must be live before the slow startup work below. + if effective_settings_app is not None: + stream_manager._init_settings_ui_if_needed() + + go_to_sleep_lock = threading.Lock() + go_to_sleep_requested = threading.Event() + + def go_to_sleep_and_stop_app() -> dict[str, Any]: + """Put Reachy to sleep, then stop the current app.""" + if not go_to_sleep_lock.acquire(blocking=False): + return {"status": "already_requested"} + + try: + if go_to_sleep_requested.is_set(): + return {"status": "already_requested"} + go_to_sleep_requested.set() + + logger.info("Going to sleep before stopping conversation app.") + sleep_error: str | None = None + + try: + robot.disable_wobbling() + except Exception as e: + logger.debug("Error disabling wobbling before sleep: %s", e) + + movement_manager.stop(reset_to_neutral=False) + + try: + robot.goto_sleep() + except Exception as e: + sleep_error = f"{type(e).__name__}: {e}" + logger.error("Failed to move Reachy Mini to sleep pose: %s", e) + + stop_current_app_requested = False + if app_stop_event is None or not app_stop_event.is_set(): + stop_current_app_requested = app_lifecycle.request_stop_current_app(robot, logger) + local_stop_requested = True + if app_stop_event is not None: + app_stop_event.set() + else: + try: + stream_manager.close() + except Exception as e: + local_stop_requested = False + logger.error("Error while closing stream manager after go_to_sleep: %s", e) + + result: dict[str, Any] = { + "status": "sleeping" if sleep_error is None else "stop_requested", + "stop_current_app_requested": stop_current_app_requested, + "local_stop_requested": local_stop_requested, + } + if sleep_error is not None: + result["error"] = f"go_to_sleep movement failed: {sleep_error}" + return result + finally: + go_to_sleep_lock.release() + + deps.go_to_sleep = go_to_sleep_and_stop_app + + def run_go_to_sleep_tool() -> dict[str, Any]: + return app_lifecycle.run_go_to_sleep_tool(deps, logger) + + if args.ui and settings_app is None and effective_settings_app is not None: + import uvicorn + + own_ui_server = uvicorn.Server( + uvicorn.Config(effective_settings_app, host="0.0.0.0", port=7860, log_level="warning") + ) + threading.Thread(target=own_ui_server.run, daemon=True, name="ui-server").start() + logger.info("Web UI available at http://localhost:7860") + + try: + initialize_tools(instance_path=instance_path) + except Exception as e: + logger.error("Failed to initialize tools: %s", e) + sys.exit(1) + + # Each async service → its own thread/loop + movement_manager.start() + # Audio-reactive head motion is driven by the daemon's wobbler, which + # taps the media pipeline at push_audio_sample. The console stream pushes + # assistant audio through that pipeline directly. + robot.enable_wobbling() + + timeout_minutes = resolve_app_timeout_minutes() + if timeout_minutes is not None: + _start_inactivity_timeout_thread(timeout_minutes, stream_manager, logger, app_stop_event, run_go_to_sleep_tool) + + def poll_stop_event() -> None: + """Poll the stop event to allow graceful shutdown.""" + if app_stop_event is not None: + app_stop_event.wait() + + logger.info("App stop event detected, shutting down...") + run_go_to_sleep_tool() + try: + stream_manager.close() + except Exception as e: + logger.error(f"Error while closing stream manager: {e}") + + if app_stop_event: + threading.Thread(target=poll_stop_event, daemon=True).start() + + try: + stream_manager.launch() + except KeyboardInterrupt: + logger.info("Keyboard interruption in main thread... closing server.") + finally: + if own_ui_server is not None: + own_ui_server.should_exit = True + + sleep_result = run_go_to_sleep_tool() + if "error" in sleep_result: + movement_manager.stop(reset_to_neutral=False) + try: + robot.disable_wobbling() + except Exception as e: + logger.debug(f"Error disabling wobbling during shutdown: {e}") + + # Ensure media is explicitly closed before disconnecting + try: + robot.media.close() + except Exception as e: + logger.debug(f"Error closing media during shutdown: {e}") + + # prevent connection to keep alive some threads + robot.client.disconnect() + time.sleep(1) + logger.info("Shutdown complete.") + + +class Peachy(ReachyMiniApp): # type: ignore[misc] + """Reachy Mini Apps entry point for the conversation app.""" + + custom_app_url = "http://0.0.0.0:7860/" + dont_start_webserver = False + + def run(self, reachy_mini: ReachyMini, stop_event: threading.Event) -> None: + """Run the Reachy Mini conversation app.""" + asyncio.set_event_loop(asyncio.new_event_loop()) + + args, _ = parse_args() + + state_root = resolve_state_root() + run( + args, + robot=reachy_mini, + app_stop_event=stop_event, + settings_app=self.settings_app, + instance_path=str(state_root), + ) + + +if __name__ == "__main__": + app = Peachy() + try: + app.wrapped_run() + except KeyboardInterrupt: + app.stop() diff --git a/src/peachy/mcp_client.py b/src/peachy/mcp_client.py new file mode 100644 index 0000000000000000000000000000000000000000..5f31cdf64c2c1a74bd2d5ab20db0adc750ce3742 --- /dev/null +++ b/src/peachy/mcp_client.py @@ -0,0 +1,391 @@ +"""Helpers for consuming remote MCP tools over HTTP(S). + +This module validates remote endpoints, discovers tools, and maps calls/results +into the app's tool interface without mutating the local project environment or +downloading third-party Python code. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import re +from typing import TYPE_CHECKING, Any, Mapping, Sequence, AsyncIterator +from datetime import timedelta +from contextlib import asynccontextmanager +from dataclasses import field, dataclass +from urllib.parse import urlparse + + +if TYPE_CHECKING: + from mcp import ClientSession + from mcp.types import Tool as McpTool + from mcp.types import CallToolResult as McpCallToolResult + + +_NAME_SEGMENT_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_NAME_NORMALIZER_PATTERN = re.compile(r"[^A-Za-z0-9_]+") +_LOCAL_HTTP_HOSTS = {"127.0.0.1", "localhost", "::1"} +_NAMESPACE_SEPARATOR = "__" + + +class McpClientError(RuntimeError): + """Base error for the MCP client.""" + + +class McpDependencyError(McpClientError): + """Raised when a required MCP client dependency is not installed.""" + + +class McpTransportError(McpClientError): + """Raised when discovery fails before a remote tool runs.""" + + +class McpToolInvocationError(McpClientError): + """Raised when a remote tool call fails at the transport layer.""" + + +class McpToolTimeoutError(McpToolInvocationError): + """Raised when a remote tool call exceeds the configured timeout.""" + + +def _require_name_segment(label: str, value: str) -> str: + candidate = value.strip() + if _NAME_SEGMENT_PATTERN.fullmatch(candidate) is None: + raise ValueError(f"Invalid {label} '{value}'. Expected pattern '[A-Za-z_][A-Za-z0-9_]*'.") + return candidate + + +def apply_name_normalization(value: str) -> str: + """Replace non-identifier characters with underscores and collapse runs.""" + normalized = _NAME_NORMALIZER_PATTERN.sub("_", value).strip("_") + return re.sub(r"_+", "_", normalized) + + +def _normalize_name_segment(label: str, value: str) -> str: + raw = value.strip() + if not raw: + raise ValueError(f"{label.capitalize()} cannot be empty.") + + normalized = apply_name_normalization(raw) + if not normalized: + raise ValueError(f"{label.capitalize()} '{value}' cannot be normalized into a valid tool identifier.") + if normalized[0].isdigit(): + normalized = f"tool_{normalized}" + return _require_name_segment(label, normalized) + + +def validate_http_mcp_url(url: str) -> str: + """Validate that the MCP endpoint uses HTTP(S).""" + parsed = urlparse(url) + if parsed.scheme not in {"http", "https"}: + raise ValueError(f"Unsupported MCP URL scheme '{parsed.scheme}'. Use http:// or https://.") + if not parsed.netloc: + raise ValueError(f"Invalid MCP URL '{url}'. Missing host.") + + host = (parsed.hostname or "").lower() + if parsed.scheme == "http" and host not in _LOCAL_HTTP_HOSTS: + raise ValueError("Remote MCP servers must use HTTPS. Plain HTTP is only allowed for localhost.") + return url + + +def build_namespaced_tool_name(server_alias: str, tool_name: str) -> str: + """Build a local tool name for a remote MCP tool.""" + alias = _require_name_segment("server alias", server_alias) + tool_segment = _normalize_name_segment("tool name", tool_name) + return f"{alias}{_NAMESPACE_SEPARATOR}{tool_segment}" + + +def _dump_content_block(block: object) -> dict[str, Any]: + if hasattr(block, "model_dump"): + dumped = block.model_dump(mode="json", by_alias=True, exclude_none=True) + if isinstance(dumped, dict): + return dumped + return {"type": getattr(block, "type", "unknown")} + + +def _join_text_content(content_blocks: list[dict[str, Any]]) -> str | None: + text_parts = [ + block["text"] for block in content_blocks if block.get("type") == "text" and isinstance(block.get("text"), str) + ] + if not text_parts: + return None + return "\n\n".join(text_parts) + + +def _exception_contains_timeout(exc: BaseException) -> bool: + timeout_exception = _httpx_timeout_exception_type() + if isinstance(exc, timeout_exception): + return True + + if "timed out" in str(exc).lower() or "deadline exceeded" in str(exc).lower(): + return True + + nested: list[BaseException] = [] + grouped_exceptions = getattr(exc, "exceptions", None) + if isinstance(grouped_exceptions, tuple): + nested.extend(grouped_exceptions) + if exc.__cause__ is not None: + nested.append(exc.__cause__) + if exc.__context__ is not None: + nested.append(exc.__context__) + + return any(_exception_contains_timeout(item) for item in nested) + + +def _load_mcp_sdk() -> tuple[type["ClientSession"], Any]: + try: + from mcp import ClientSession + from mcp.client.streamable_http import streamable_http_client + except ImportError as exc: + raise McpDependencyError( + "Remote MCP tools require the app's MCP client dependencies. Reinstall or update the app environment." + ) from exc + return ClientSession, streamable_http_client + + +def _load_httpx() -> Any: + try: + import httpx + except ImportError as exc: + raise McpDependencyError( + "Remote MCP tools require the app's HTTP client dependencies. Reinstall or update the app environment." + ) from exc + return httpx + + +def _httpx_timeout_exception_type() -> tuple[type[BaseException], ...]: + try: + timeout_exception = _load_httpx().TimeoutException + except McpDependencyError: + return (TimeoutError,) + return (TimeoutError, timeout_exception) + + +@dataclass(frozen=True) +class RemoteMcpServerConfig: + """Allowlisted MCP server configuration.""" + + alias: str + url: str + headers: Mapping[str, str] = field(default_factory=dict) + request_timeout_s: float = 10.0 + tool_timeout_s: float = 30.0 + + def __post_init__(self) -> None: + """Validate configuration once the dataclass has been created.""" + object.__setattr__(self, "alias", _require_name_segment("server alias", self.alias)) + object.__setattr__(self, "url", validate_http_mcp_url(self.url)) + object.__setattr__(self, "headers", {str(k): str(v) for k, v in self.headers.items()}) + if self.request_timeout_s <= 0: + raise ValueError("request_timeout_s must be greater than zero.") + if self.tool_timeout_s <= 0: + raise ValueError("tool_timeout_s must be greater than zero.") + + +@dataclass(frozen=True) +class RemoteToolSpec: + """App-facing representation of a remote MCP tool.""" + + server_alias: str + remote_name: str + namespaced_name: str + description: str + parameters_schema: dict[str, Any] + + @classmethod + def from_mcp_tool(cls, server_alias: str, tool: "McpTool") -> "RemoteToolSpec": + """Build an app-facing spec from an MCP SDK tool descriptor.""" + description = (getattr(tool, "description", None) or "").strip() + parameters_schema = getattr(tool, "inputSchema", None) + if not isinstance(parameters_schema, dict): + parameters_schema = {"type": "object", "properties": {}, "required": []} + + remote_name = str(getattr(tool, "name", "")).strip() + if not remote_name: + raise ValueError("Remote MCP tool is missing a name.") + + return cls( + server_alias=server_alias, + remote_name=remote_name, + namespaced_name=build_namespaced_tool_name(server_alias, remote_name), + description=description or f"Remote MCP tool '{remote_name}' from server '{server_alias}'.", + parameters_schema=dict(parameters_schema), + ) + + def to_function_spec(self) -> dict[str, Any]: + """Translate to the app's function-calling shape.""" + return { + "type": "function", + "name": self.namespaced_name, + "description": self.description, + "parameters": self.parameters_schema, + } + + +@dataclass(frozen=True) +class RemoteToolCallResponse: + """Mapped result for a remote MCP tool call.""" + + server_alias: str + remote_tool_name: str + namespaced_tool_name: str + status: str + content_blocks: list[dict[str, Any]] + text: str | None + structured_content: Any | None + + @classmethod + def from_call_tool_result( + cls, + *, + server_alias: str, + remote_tool_name: str, + result: "McpCallToolResult", + ) -> "RemoteToolCallResponse": + """Convert an MCP SDK tool result into the app's result envelope.""" + content_blocks = [_dump_content_block(block) for block in getattr(result, "content", [])] + return cls( + server_alias=server_alias, + remote_tool_name=remote_tool_name, + namespaced_tool_name=build_namespaced_tool_name(server_alias, remote_tool_name), + status="error" if bool(getattr(result, "isError", False)) else "ok", + content_blocks=content_blocks, + text=_join_text_content(content_blocks), + structured_content=getattr(result, "structuredContent", None), + ) + + def to_tool_result(self) -> dict[str, Any]: + """Return a dict shaped like the app's tool results.""" + payload: dict[str, Any] = { + "status": self.status, + "server_alias": self.server_alias, + "remote_tool_name": self.remote_tool_name, + "namespaced_tool_name": self.namespaced_tool_name, + "content_blocks": self.content_blocks, + } + if self.text is not None: + payload["text"] = self.text + if self.structured_content is not None: + payload["structured_content"] = self.structured_content + return payload + + +class RemoteMcpToolClient: + """Minimal async client for allowlisted remote MCP tool servers.""" + + def __init__(self, server: RemoteMcpServerConfig, known_tools: Sequence[RemoteToolSpec] = ()) -> None: + """Store one allowlisted server configuration and an in-memory tool cache.""" + self.server = server + self._tool_index = _index_remote_tools(list(known_tools)) + + async def list_tool_specs(self) -> list[RemoteToolSpec]: + """Discover remote tools and translate them into app-facing specs.""" + try: + async with self._session() as session: + discovered = await self._list_all_tools(session) + except McpDependencyError: + raise + except Exception as exc: + raise McpTransportError( + f"Failed to discover MCP tools from '{self.server.alias}' at {self.server.url}: {exc}" + ) from exc + + specs = [RemoteToolSpec.from_mcp_tool(self.server.alias, tool) for tool in discovered] + self._tool_index = _index_remote_tools(specs) + return specs + + async def list_function_specs(self) -> list[dict[str, Any]]: + """Discover tools and translate them into function-calling specs.""" + return [spec.to_function_spec() for spec in await self.list_tool_specs()] + + async def call_tool(self, namespaced_tool_name: str, arguments: Mapping[str, Any] | None = None) -> dict[str, Any]: + """Invoke a remote MCP tool by its namespaced local ID.""" + spec = await self._resolve_tool_spec(namespaced_tool_name) + timeout_exception = _httpx_timeout_exception_type() + + try: + async with self._session() as session: + result = await session.call_tool( + spec.remote_name, + arguments=dict(arguments or {}), + read_timeout_seconds=timedelta(seconds=self.server.tool_timeout_s), + ) + except McpDependencyError: + raise + except timeout_exception as exc: + raise McpToolTimeoutError( + f"Timed out calling MCP tool '{namespaced_tool_name}' from '{self.server.alias}'." + ) from exc + except Exception as exc: + if _exception_contains_timeout(exc): + raise McpToolTimeoutError( + f"Timed out calling MCP tool '{namespaced_tool_name}' from '{self.server.alias}'." + ) from exc + raise McpToolInvocationError( + f"Failed to call MCP tool '{namespaced_tool_name}' from '{self.server.alias}': {exc}" + ) from exc + + return RemoteToolCallResponse.from_call_tool_result( + server_alias=self.server.alias, + remote_tool_name=spec.remote_name, + result=result, + ).to_tool_result() + + async def _resolve_tool_spec(self, namespaced_tool_name: str) -> RemoteToolSpec: + spec = self._tool_index.get(namespaced_tool_name) + if spec is not None: + return spec + + await self.list_tool_specs() + spec = self._tool_index.get(namespaced_tool_name) + if spec is None: + raise ValueError(f"Unknown remote MCP tool '{namespaced_tool_name}' for server '{self.server.alias}'.") + return spec + + async def _list_all_tools(self, session: "ClientSession") -> list["McpTool"]: + tools: list[McpTool] = [] + cursor: str | None = None + while True: + page = await session.list_tools(cursor=cursor) + tools.extend(page.tools) + cursor = getattr(page, "nextCursor", None) + if cursor is None: + return tools + + @asynccontextmanager + async def _session(self) -> AsyncIterator["ClientSession"]: + client_session_cls, streamable_http_client = _load_mcp_sdk() + httpx = _load_httpx() + client_timeout = max(self.server.request_timeout_s, self.server.tool_timeout_s) + + async with httpx.AsyncClient( + headers=self.server.headers, + follow_redirects=False, + timeout=client_timeout, + ) as http_client: + async with streamable_http_client(self.server.url, http_client=http_client) as transport: + read_stream, write_stream, _ = transport + async with client_session_cls(read_stream, write_stream) as session: + await session.initialize() + yield session + + +def _index_remote_tools(specs: list[RemoteToolSpec]) -> dict[str, RemoteToolSpec]: + index: dict[str, RemoteToolSpec] = {} + collisions: dict[str, list[str]] = {} + + for spec in specs: + existing = index.get(spec.namespaced_name) + if existing is None: + index[spec.namespaced_name] = spec + continue + + collisions.setdefault(spec.namespaced_name, [existing.remote_name]).append(spec.remote_name) + + if collisions: + details = "; ".join( + f"{tool_name}: {sorted(remote_names)}" for tool_name, remote_names in sorted(collisions.items()) + ) + raise ValueError(f"Remote MCP tool names collide after local namespacing/normalization. Conflicts: {details}") + + return index diff --git a/src/peachy/memory.py b/src/peachy/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..c53a21ad441405935375ffe46a983ff0540cc87d --- /dev/null +++ b/src/peachy/memory.py @@ -0,0 +1,213 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +from __future__ import annotations +import os +import json +import time +import random +import string +import logging +import threading +from pathlib import Path +from dataclasses import dataclass +from collections.abc import Mapping + + +logger = logging.getLogger(__name__) + +SCHEMA_VERSION = 1 +MAX_FACTS = 60 +MAX_FACT_CHARS = 280 +MEMORY_FILENAME = "memory.v1.json" + +_STORE_LOCK = threading.Lock() + + +@dataclass(frozen=True) +class MemoryFact: + """One short long-term memory fact.""" + + id: str + text: str + created_at: int + + def to_json(self) -> dict[str, object]: + """Return the persisted JSON shape used by the mobile app.""" + return { + "id": self.id, + "text": self.text, + "createdAt": self.created_at, + } + + +@dataclass(frozen=True) +class ForgetMemoryResult: + """Result of removing a memory fact.""" + + removed: MemoryFact | None + candidates: tuple[MemoryFact, ...] + + +def memory_path_for_instance(instance_path: str | Path | None = None) -> Path: + """Return the memory JSON path for this app instance.""" + if instance_path is not None: + return Path(instance_path).expanduser() / MEMORY_FILENAME + + data_home = os.getenv("XDG_DATA_HOME") + data_root = Path(data_home).expanduser() if data_home else Path.home() / ".local" / "share" + return data_root / "peachy" / MEMORY_FILENAME + + +def normalize_memory_text(text: str) -> str: + """Collapse whitespace and enforce the fact length cap.""" + normalized = " ".join(text.split()).strip() + if len(normalized) <= MAX_FACT_CHARS: + return normalized + return f"{normalized[: MAX_FACT_CHARS - 3]}..." + + +def _make_id() -> str: + suffix = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) + return f"m_{int(time.time() * 1000)}_{suffix}" + + +def _now_ms() -> int: + return int(time.time() * 1000) + + +def _fact_from_json(value: object) -> MemoryFact | None: + if not isinstance(value, Mapping): + return None + + fact_id = value.get("id") + text = value.get("text") + created_at = value.get("createdAt") + + if not isinstance(fact_id, str): + return None + if not isinstance(text, str): + return None + if not isinstance(created_at, (int, float)): + return None + + normalized = normalize_memory_text(text) + if not normalized: + return None + + return MemoryFact(id=fact_id, text=normalized, created_at=int(created_at)) + + +def _read_memory_file(path: Path) -> list[MemoryFact]: + try: + raw = path.read_text(encoding="utf-8") + except FileNotFoundError: + return [] + except OSError as exc: + logger.warning("Failed to read memory store at %s: %s", path, exc) + return [] + + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + logger.warning("Failed to parse memory store at %s: %s", path, exc) + return [] + + if not isinstance(parsed, Mapping): + return [] + + facts_value = parsed.get("facts") + if not isinstance(facts_value, list): + return [] + + facts: list[MemoryFact] = [] + for item in facts_value: + fact = _fact_from_json(item) + if fact is not None: + facts.append(fact) + return facts[:MAX_FACTS] + + +def _write_memory_file(path: Path, facts: list[MemoryFact]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "version": SCHEMA_VERSION, + "facts": [fact.to_json() for fact in facts[:MAX_FACTS]], + } + tmp_path = path.with_name(f".{path.name}.{os.getpid()}.tmp") + try: + tmp_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + tmp_path.replace(path) + finally: + try: + tmp_path.unlink(missing_ok=True) + except OSError: + pass + + +def list_memory_facts(instance_path: str | Path | None = None) -> list[MemoryFact]: + """Return stored memory facts, newest first.""" + with _STORE_LOCK: + return list(_read_memory_file(memory_path_for_instance(instance_path))) + + +def add_memory_fact(instance_path: str | Path | None, text: str) -> MemoryFact | None: + """Store one short fact, deduplicating exact case-insensitive matches.""" + normalized = normalize_memory_text(text) + if not normalized: + return None + + path = memory_path_for_instance(instance_path) + with _STORE_LOCK: + facts = _read_memory_file(path) + existing = next((fact for fact in facts if fact.text.lower() == normalized.lower()), None) + if existing is not None: + return existing + + fact = MemoryFact(id=_make_id(), text=normalized, created_at=_now_ms()) + _write_memory_file(path, [fact, *facts][:MAX_FACTS]) + return fact + + +def forget_memory_fact( + instance_path: str | Path | None, + *, + query: str | None = None, +) -> ForgetMemoryResult: + """Remove a fact by case-insensitive substring query.""" + path = memory_path_for_instance(instance_path) + with _STORE_LOCK: + facts = _read_memory_file(path) + + normalized_query = normalize_memory_text(query or "").lower() + if not normalized_query: + return ForgetMemoryResult(removed=None, candidates=()) + + candidates = tuple(fact for fact in facts if normalized_query in fact.text.lower()) + if not candidates: + return ForgetMemoryResult(removed=None, candidates=()) + + removed = candidates[0] + _write_memory_file(path, [fact for fact in facts if fact.id != removed.id]) + return ForgetMemoryResult(removed=removed, candidates=candidates) + + +def clear_memory_facts(instance_path: str | Path | None = None) -> None: + """Remove all stored memory facts.""" + path = memory_path_for_instance(instance_path) + with _STORE_LOCK: + _write_memory_file(path, []) + + +def format_memory_for_prompt(instance_path: str | Path | None = None) -> str: + """Return the prompt fragment injected before the session instructions.""" + facts = list_memory_facts(instance_path) + if not facts: + return "" + + bullets = "\n".join(f"- {fact.text}" for fact in facts) + return "\n".join( + [ + "Things you remember about the user (use this context naturally,", + "do not recite the list verbatim):", + bullets, + ] + ) diff --git a/src/peachy/moves.py b/src/peachy/moves.py new file mode 100644 index 0000000000000000000000000000000000000000..f64f68dec98c9ca281fea82a13df69b5483859f4 --- /dev/null +++ b/src/peachy/moves.py @@ -0,0 +1,767 @@ +"""Movement system driving sequential primary moves. + +Design overview +- Primary moves (emotions, dances, goto, breathing) are mutually exclusive and run + sequentially. +- There is a single control point to the robot: `ReachyMini.set_target`. +- The control loop runs near 100 Hz and is phase-aligned via a monotonic clock. +- Idle behaviour starts an infinite `BreathingMove` after a short inactivity delay + unless listening is active. + +Threading model +- A dedicated worker thread owns all real-time state and issues `set_target` + commands. +- Other threads communicate via a command queue (enqueue moves, mark activity, + toggle listening). + +Units and frames +- Antennas and `body_yaw` are in radians. + +Safety +- Listening freezes antennas, then blends them back on unfreeze. +- Interpolations and blends are used to avoid jumps at all times. +- `set_target` errors are rate-limited in logs. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import time +import logging +import threading +from queue import Empty, Queue +from typing import Any, Dict, Tuple +from collections import deque +from dataclasses import dataclass + +import numpy as np +from numpy.typing import NDArray + +from reachy_mini import ReachyMini +from reachy_mini.utils import create_head_pose +from reachy_mini.motion.move import Move +from reachy_mini.utils.interpolation import compose_world_offset, linear_pose_interpolation +from peachy.dance_emotion_moves import EmotionQueueMove + + +logger = logging.getLogger(__name__) + +# Configuration constants +CONTROL_LOOP_FREQUENCY_HZ = 60.0 # Hz - Target frequency for the movement control loop + +# Type definitions +FullBodyPose = Tuple[NDArray[np.float32], Tuple[float, float], float] # (head_pose_4x4, antennas, body_yaw) + + +class BreathingMove(Move): # type: ignore + """Breathing move with interpolation to neutral and then continuous breathing patterns.""" + + def __init__( + self, + interpolation_start_pose: NDArray[np.float32], + interpolation_start_antennas: Tuple[float, float], + interpolation_duration: float = 1.0, + ): + """Initialize breathing move. + + Args: + interpolation_start_pose: 4x4 matrix of current head pose to interpolate from + interpolation_start_antennas: Current antenna positions to interpolate from + interpolation_duration: Duration of interpolation to neutral (seconds) + + """ + self.interpolation_start_pose = interpolation_start_pose + self.interpolation_start_antennas = np.array(interpolation_start_antennas) + self.interpolation_duration = interpolation_duration + + # Neutral positions for breathing base + self.neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + self.neutral_antennas = np.array([-0.1745, 0.1745]) # ~10° offset to reduce shaking + + # Breathing parameters + self.breathing_z_amplitude = 0.005 # 5mm gentle breathing + self.breathing_frequency = 0.1 # Hz (6 breaths per minute) + self.antenna_sway_amplitude = np.deg2rad(15) # 15 degrees + self.antenna_frequency = 0.5 # Hz (faster antenna sway) + + @property + def duration(self) -> float: + """Duration property required by official Move interface.""" + return float("inf") # Continuous breathing (never ends naturally) + + def evaluate(self, t: float) -> tuple[NDArray[np.float64] | None, NDArray[np.float64] | None, float | None]: + """Evaluate breathing move at time t.""" + if t < self.interpolation_duration: + # Phase 1: Interpolate to neutral base position + interpolation_t = t / self.interpolation_duration + + # Interpolate head pose + head_pose = linear_pose_interpolation( + self.interpolation_start_pose, + self.neutral_head_pose, + interpolation_t, + ) + + # Interpolate antennas + antennas_interp = ( + 1 - interpolation_t + ) * self.interpolation_start_antennas + interpolation_t * self.neutral_antennas + antennas = antennas_interp.astype(np.float64) + + else: + # Phase 2: Breathing patterns from neutral base + breathing_time = t - self.interpolation_duration + + # Gentle z-axis breathing + z_offset = self.breathing_z_amplitude * np.sin(2 * np.pi * self.breathing_frequency * breathing_time) + head_pose = create_head_pose(x=0, y=0, z=z_offset, roll=0, pitch=0, yaw=0, degrees=True, mm=False) + + # Antenna sway (opposite directions) + antenna_sway = self.antenna_sway_amplitude * np.sin(2 * np.pi * self.antenna_frequency * breathing_time) + antennas = np.array([antenna_sway, -antenna_sway], dtype=np.float64) + + # Return in official Move interface format: (head_pose, antennas_array, body_yaw) + return (head_pose, antennas, 0.0) + + +def clone_full_body_pose(pose: FullBodyPose) -> FullBodyPose: + """Create a deep copy of a full body pose tuple.""" + head, antennas, body_yaw = pose + return (head.copy(), (float(antennas[0]), float(antennas[1])), float(body_yaw)) + + +@dataclass +class MovementState: + """State tracking for the movement system.""" + + # Primary move state + current_move: Move | None = None + move_start_time: float | None = None + last_activity_time: float = 0.0 + + # Status flags + last_primary_pose: FullBodyPose | None = None + + def update_activity(self) -> None: + """Update the last activity time.""" + self.last_activity_time = time.monotonic() + + +@dataclass +class LoopFrequencyStats: + """Track rolling loop frequency statistics.""" + + mean: float = 0.0 + m2: float = 0.0 + min_freq: float = float("inf") + count: int = 0 + last_freq: float = 0.0 + potential_freq: float = 0.0 + + def reset(self) -> None: + """Reset accumulators while keeping the last potential frequency.""" + self.mean = 0.0 + self.m2 = 0.0 + self.min_freq = float("inf") + self.count = 0 + + +class MovementManager: + """Coordinate sequential moves and robot output at 100 Hz. + + Responsibilities: + - Own a real-time loop that samples the current primary move (if any) and calls + `set_target` exactly once per tick. + - Start an idle `BreathingMove` after `idle_inactivity_delay` when not + listening and no moves are queued. + - Expose thread-safe APIs so other threads can enqueue moves or mark activity + without touching internal state. + + Timing: + - All elapsed-time calculations rely on `time.monotonic()` through `self._now` + to avoid wall-clock jumps. + - The loop attempts 100 Hz + + Concurrency: + - External threads communicate via `_command_queue` messages. + """ + + def __init__( + self, + current_robot: ReachyMini, + ): + """Initialize movement manager.""" + self.current_robot = current_robot + self._head_tracking = False + + # Single timing source for durations + self._now = time.monotonic + + # Movement state + self.state = MovementState() + self.state.last_activity_time = self._now() + neutral_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + self.state.last_primary_pose = (neutral_pose, (0.0, 0.0), 0.0) + + # Move queue (primary moves) + self.move_queue: deque[Move] = deque() + + # Configuration + self.idle_inactivity_delay = 0.3 # seconds + self.target_frequency = CONTROL_LOOP_FREQUENCY_HZ + self.target_period = 1.0 / self.target_frequency + + self._stop_event = threading.Event() + self._thread: threading.Thread | None = None + self._is_listening = False + # Speaking pauses tracking; the captured look-at pose anchors queued moves. + self._is_speaking = False + self._track_anchor: NDArray[np.float64] | None = None + self._last_commanded_pose: FullBodyPose = clone_full_body_pose(self.state.last_primary_pose) + self._listening_antennas: Tuple[float, float] = self._last_commanded_pose[1] + self._antenna_unfreeze_blend = 1.0 + self._antenna_blend_duration = 0.4 # seconds to blend back after listening + self._last_listening_blend_time = self._now() + self._breathing_active = False # true when breathing move is running or queued + self._listening_debounce_s = 0.15 + self._last_listening_toggle_time = self._now() + self._last_set_target_err = 0.0 + self._set_target_err_interval = 1.0 # seconds between error logs + self._set_target_err_suppressed = 0 + + # Cross-thread signalling + self._command_queue: "Queue[Tuple[str, Any]]" = Queue() + + self._shared_state_lock = threading.Lock() + self._shared_last_activity_time = self.state.last_activity_time + self._shared_is_listening = self._is_listening + self._status_lock = threading.Lock() + self._freq_stats = LoopFrequencyStats() + self._freq_snapshot = LoopFrequencyStats() + + def queue_move(self, move: Move) -> None: + """Queue a primary move to run after the currently executing one. + + Thread-safe: the move is enqueued via the worker command queue so the + control loop remains the sole mutator of movement state. + """ + self._command_queue.put(("queue_move", move)) + + def clear_move_queue(self) -> None: + """Stop the active move and discard any queued primary moves. + + Thread-safe: executed by the worker thread via the command queue. + """ + self._command_queue.put(("clear_queue", None)) + + def set_moving_state(self, duration: float) -> None: + """Mark the robot as actively moving for the provided duration. + + Legacy hook used by goto helpers to keep inactivity and breathing logic + aware of manual motions. Thread-safe via the command queue. + """ + self._command_queue.put(("set_moving_state", duration)) + + def is_idle(self) -> bool: + """Return True when the robot has been inactive longer than the idle delay.""" + with self._shared_state_lock: + last_activity = self._shared_last_activity_time + listening = self._shared_is_listening + + if listening: + return False + + return self._now() - last_activity >= self.idle_inactivity_delay + + def set_listening(self, listening: bool) -> None: + """Enable or disable listening mode without touching shared state directly. + + While listening: + - Antenna positions are frozen at the last commanded values. + - Blending is reset so that upon unfreezing the antennas return smoothly. + - Idle breathing is suppressed. + + Thread-safe: the change is posted to the worker command queue. + """ + with self._shared_state_lock: + if self._shared_is_listening == listening: + return + self._command_queue.put(("set_listening", listening)) + + def set_head_tracking(self, enabled: bool) -> None: + """Start or stop following the user's face; thread-safe via the command queue.""" + self._command_queue.put(("set_head_tracking", enabled)) + + def set_speaking(self, speaking: bool) -> None: + """Pause head tracking while the assistant speaks, resume it afterwards. + + On speaking, the current look-at pose is captured as an anchor so queued + moves stay on the user; when speaking stops the head is handed back to + daemon tracking. Thread-safe: posted to the worker command queue. + """ + self._command_queue.put(("set_speaking", speaking)) + + def _poll_signals(self, current_time: float) -> None: + """Apply queued commands.""" + while True: + try: + command, payload = self._command_queue.get_nowait() + except Empty: + break + self._handle_command(command, payload, current_time) + + def _handle_command(self, command: str, payload: Any, current_time: float) -> None: + """Handle a single cross-thread command.""" + if command == "queue_move": + if isinstance(payload, Move): + self.move_queue.append(payload) + self.state.update_activity() + duration = getattr(payload, "duration", None) + if duration is not None: + try: + duration_str = f"{float(duration):.2f}" + except (TypeError, ValueError): + duration_str = str(duration) + else: + duration_str = "?" + logger.debug( + "Queued move with duration %ss, queue size: %s", + duration_str, + len(self.move_queue), + ) + else: + logger.warning("Ignored queue_move command with invalid payload: %s", payload) + elif command == "clear_queue": + self.move_queue.clear() + self.state.current_move = None + self.state.move_start_time = None + self._breathing_active = False + logger.info("Cleared move queue and stopped current move") + elif command == "set_moving_state": + try: + duration = float(payload) + except (TypeError, ValueError): + logger.warning("Invalid moving state duration: %s", payload) + return + self.state.update_activity() + elif command == "mark_activity": + self.state.update_activity() + elif command == "set_listening": + desired_state = bool(payload) + now = self._now() + if now - self._last_listening_toggle_time < self._listening_debounce_s: + return + self._last_listening_toggle_time = now + + if self._is_listening == desired_state: + return + + self._is_listening = desired_state + self._last_listening_blend_time = now + if desired_state: + # Freeze: snapshot current commanded antennas and reset blend + self._listening_antennas = ( + float(self._last_commanded_pose[1][0]), + float(self._last_commanded_pose[1][1]), + ) + self._antenna_unfreeze_blend = 0.0 + else: + # Unfreeze: restart blending from frozen pose + self._antenna_unfreeze_blend = 0.0 + self.state.update_activity() + elif command == "set_head_tracking": + enabled = bool(payload) + if self._head_tracking == enabled: + return + self._head_tracking = enabled + self._track_anchor = None + # set_speaking is gated off while disabled, so its state would go stale across a toggle. + self._is_speaking = False + try: + if enabled: + self.current_robot.start_head_tracking(weight=1.0) + else: + self.current_robot.stop_head_tracking() + except Exception as e: + logger.warning("Head-tracking toggle failed: %s", e) + elif command == "set_speaking": + if not self._head_tracking: + return + speaking = bool(payload) + if self._is_speaking == speaking: + return + self._is_speaking = speaking + try: + if speaking and self.current_robot.get_tracked_face(wait=False).detected: + # Pause only once a face is locked, else speech blocks acquisition. + self._track_anchor = self.current_robot.get_current_head_pose() + self.current_robot.start_head_tracking(weight=0.0) + elif not speaking: + self._track_anchor = None + self.current_robot.start_head_tracking(weight=1.0) + except Exception as e: + logger.warning("Head-tracking speaking handoff failed: %s", e) + else: + logger.warning("Unknown command received by MovementManager: %s", command) + + def _publish_shared_state(self) -> None: + """Expose idle-related state for external threads.""" + with self._shared_state_lock: + self._shared_last_activity_time = self.state.last_activity_time + self._shared_is_listening = self._is_listening + + def _manage_move_queue(self, current_time: float) -> None: + """Manage the primary move queue (sequential execution).""" + if self.state.current_move is None or ( + self.state.move_start_time is not None + and current_time - self.state.move_start_time >= self.state.current_move.duration + ): + self.state.current_move = None + self.state.move_start_time = None + + if self.move_queue: + self.state.current_move = self.move_queue.popleft() + self.state.move_start_time = current_time + # Any real move cancels breathing mode flag + self._breathing_active = isinstance(self.state.current_move, BreathingMove) + logger.debug(f"Starting new move, duration: {self.state.current_move.duration}s") + + def _manage_breathing(self, current_time: float) -> None: + """Manage automatic breathing when idle.""" + if ( + self.state.current_move is None + and not self.move_queue + and not self._is_listening + and not self._breathing_active + ): + idle_for = current_time - self.state.last_activity_time + if idle_for >= self.idle_inactivity_delay: + try: + # These 2 functions return the latest available sensor data from the robot, but don't perform I/O synchronously. + # Therefore, we accept calling them inside the control loop. + _, current_antennas = self.current_robot.get_current_joint_positions() + current_head_pose = self.current_robot.get_current_head_pose() + + self._breathing_active = True + self.state.update_activity() + + breathing_move = BreathingMove( + interpolation_start_pose=current_head_pose, + interpolation_start_antennas=current_antennas, + interpolation_duration=1.0, + ) + self.move_queue.append(breathing_move) + logger.debug("Started breathing after %.1fs of inactivity", idle_for) + except Exception as e: + self._breathing_active = False + logger.error("Failed to start breathing: %s", e) + + if isinstance(self.state.current_move, BreathingMove) and self.move_queue: + self.state.current_move = None + self.state.move_start_time = None + self._breathing_active = False + logger.debug("Stopping breathing due to new move activity") + + if self.state.current_move is not None and not isinstance(self.state.current_move, BreathingMove): + self._breathing_active = False + + def _get_primary_pose(self, current_time: float) -> FullBodyPose: + """Get the primary full body pose from current move or neutral.""" + # When a primary move is playing, sample it and cache the resulting pose + if self.state.current_move is not None and self.state.move_start_time is not None: + move_time = current_time - self.state.move_start_time + head, antennas, body_yaw = self.state.current_move.evaluate(move_time) + + if head is None: + head = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + if antennas is None: + antennas = np.array([-0.1745, 0.1745]) # ~10° offset + if body_yaw is None: + body_yaw = 0.0 + + antennas_tuple = (float(antennas[0]), float(antennas[1])) + head_copy = head.copy() + primary_full_body_pose = ( + head_copy, + antennas_tuple, + float(body_yaw), + ) + + self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose) + # Otherwise reuse the last primary pose so we avoid jumps between moves + elif self.state.last_primary_pose is not None: + primary_full_body_pose = clone_full_body_pose(self.state.last_primary_pose) + else: + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + primary_full_body_pose = (neutral_head_pose, (0.0, 0.0), 0.0) + self.state.last_primary_pose = clone_full_body_pose(primary_full_body_pose) + + # Speaking pauses tracking: hold the look-at anchor, overlay emotions on it, dance from neutral. + if self._track_anchor is not None: + head_pose, antennas, body_yaw = primary_full_body_pose + move = self.state.current_move + if move is None: + head_pose = self._track_anchor.copy() + elif isinstance(move, EmotionQueueMove): + head_pose = compose_world_offset(self._track_anchor, head_pose) + primary_full_body_pose = (head_pose, antennas, body_yaw) + + return primary_full_body_pose + + def _update_primary_motion(self, current_time: float) -> None: + """Advance queue state and idle behaviours for this tick.""" + self._manage_move_queue(current_time) + self._manage_breathing(current_time) + + def _calculate_blended_antennas(self, target_antennas: Tuple[float, float]) -> Tuple[float, float]: + """Blend target antennas with listening freeze state and update blending.""" + now = self._now() + listening = self._is_listening + listening_antennas = self._listening_antennas + blend = self._antenna_unfreeze_blend + blend_duration = self._antenna_blend_duration + last_update = self._last_listening_blend_time + self._last_listening_blend_time = now + + if listening: + antennas_cmd = listening_antennas + new_blend = 0.0 + else: + dt = max(0.0, now - last_update) + if blend_duration <= 0: + new_blend = 1.0 + else: + new_blend = min(1.0, blend + dt / blend_duration) + antennas_cmd = ( + listening_antennas[0] * (1.0 - new_blend) + target_antennas[0] * new_blend, + listening_antennas[1] * (1.0 - new_blend) + target_antennas[1] * new_blend, + ) + + if listening: + self._antenna_unfreeze_blend = 0.0 + else: + self._antenna_unfreeze_blend = new_blend + if new_blend >= 1.0: + self._listening_antennas = ( + float(target_antennas[0]), + float(target_antennas[1]), + ) + + return antennas_cmd + + def _issue_control_command( + self, head: NDArray[np.float32], antennas: Tuple[float, float], body_yaw: float + ) -> None: + """Send the pose to the robot with throttled error logging.""" + try: + self.current_robot.set_target(head=head, antennas=antennas, body_yaw=body_yaw) + except Exception as e: + now = self._now() + if now - self._last_set_target_err >= self._set_target_err_interval: + msg = f"Failed to set robot target: {e}" + if self._set_target_err_suppressed: + msg += f" (suppressed {self._set_target_err_suppressed} repeats)" + self._set_target_err_suppressed = 0 + logger.error(msg) + self._last_set_target_err = now + else: + self._set_target_err_suppressed += 1 + else: + with self._status_lock: + self._last_commanded_pose = clone_full_body_pose((head, antennas, body_yaw)) + + def _update_frequency_stats( + self, + loop_start: float, + prev_loop_start: float, + stats: LoopFrequencyStats, + ) -> LoopFrequencyStats: + """Update frequency statistics based on the current loop start time.""" + period = loop_start - prev_loop_start + if period > 0: + stats.last_freq = 1.0 / period + stats.count += 1 + delta = stats.last_freq - stats.mean + stats.mean += delta / stats.count + stats.m2 += delta * (stats.last_freq - stats.mean) + stats.min_freq = min(stats.min_freq, stats.last_freq) + return stats + + def _schedule_next_tick(self, loop_start: float, stats: LoopFrequencyStats) -> Tuple[float, LoopFrequencyStats]: + """Compute sleep time to maintain target frequency and update potential freq.""" + computation_time = self._now() - loop_start + stats.potential_freq = 1.0 / computation_time if computation_time > 0 else float("inf") + sleep_time = max(0.0, self.target_period - computation_time) + return sleep_time, stats + + def _record_frequency_snapshot(self, stats: LoopFrequencyStats) -> None: + """Store a thread-safe snapshot of current frequency statistics.""" + with self._status_lock: + self._freq_snapshot = LoopFrequencyStats( + mean=stats.mean, + m2=stats.m2, + min_freq=stats.min_freq, + count=stats.count, + last_freq=stats.last_freq, + potential_freq=stats.potential_freq, + ) + + def _maybe_log_frequency(self, loop_count: int, print_interval_loops: int, stats: LoopFrequencyStats) -> None: + """Emit frequency telemetry when enough loops have elapsed.""" + if loop_count % print_interval_loops != 0 or stats.count == 0: + return + + variance = stats.m2 / stats.count if stats.count > 0 else 0.0 + lowest = stats.min_freq if stats.min_freq != float("inf") else 0.0 + logger.debug( + "Loop freq - avg: %.2fHz, variance: %.4f, min: %.2fHz, last: %.2fHz, potential: %.2fHz, target: %.1fHz", + stats.mean, + variance, + lowest, + stats.last_freq, + stats.potential_freq, + self.target_frequency, + ) + stats.reset() + + def start(self) -> None: + """Start the worker thread that drives the 100 Hz control loop.""" + if self._thread is not None and self._thread.is_alive(): + logger.warning("Move worker already running; start() ignored") + return + self._stop_event.clear() + self._thread = threading.Thread(target=self.working_loop, daemon=True) + self._thread.start() + logger.debug("Move worker started") + + def stop(self, reset_to_neutral: bool = True) -> None: + """Request the worker thread to stop and wait for it to exit. + + Optionally resets the robot to a neutral position after stopping. + """ + if self._thread is None or not self._thread.is_alive(): + logger.debug("Move worker not running; stop() ignored") + return + + if reset_to_neutral: + logger.info("Stopping movement manager and resetting to neutral position...") + else: + logger.info("Stopping movement manager...") + + # Clear any queued moves and stop current move + self.clear_move_queue() + + # Stop the worker thread first so it doesn't interfere + self._stop_event.set() + if self._thread is not None: + self._thread.join() + self._thread = None + logger.debug("Move worker stopped") + + if self._head_tracking: + try: + self.current_robot.stop_head_tracking() + except Exception as e: + logger.warning("Failed to stop head tracking: %s", e) + + if not reset_to_neutral: + return + + # Reset to neutral position using goto_target (same approach as wake_up) + try: + neutral_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=True) + neutral_antennas = [-0.1745, 0.1745] # ~10° offset to reduce shaking + neutral_body_yaw = 0.0 + + # Use goto_target directly on the robot + self.current_robot.goto_target( + head=neutral_head_pose, + antennas=neutral_antennas, + duration=2.0, + body_yaw=neutral_body_yaw, + ) + + logger.info("Reset to neutral position completed") + + except Exception as e: + logger.error(f"Failed to reset to neutral position: {e}") + + def get_status(self) -> Dict[str, Any]: + """Return a lightweight status snapshot for observability.""" + with self._status_lock: + pose_snapshot = clone_full_body_pose(self._last_commanded_pose) + freq_snapshot = LoopFrequencyStats( + mean=self._freq_snapshot.mean, + m2=self._freq_snapshot.m2, + min_freq=self._freq_snapshot.min_freq, + count=self._freq_snapshot.count, + last_freq=self._freq_snapshot.last_freq, + potential_freq=self._freq_snapshot.potential_freq, + ) + + head_matrix = pose_snapshot[0].tolist() if pose_snapshot else None + antennas = pose_snapshot[1] if pose_snapshot else None + body_yaw = pose_snapshot[2] if pose_snapshot else None + + return { + "queue_size": len(self.move_queue), + "is_listening": self._is_listening, + "breathing_active": self._breathing_active, + "last_commanded_pose": { + "head": head_matrix, + "antennas": antennas, + "body_yaw": body_yaw, + }, + "loop_frequency": { + "last": freq_snapshot.last_freq, + "mean": freq_snapshot.mean, + "min": freq_snapshot.min_freq, + "potential": freq_snapshot.potential_freq, + "samples": freq_snapshot.count, + }, + } + + def working_loop(self) -> None: + """Run the primary-move control loop with a single set_target() call per tick.""" + logger.debug("Starting enhanced movement control loop (100Hz)") + + loop_count = 0 + prev_loop_start = self._now() + print_interval_loops = max(1, int(self.target_frequency * 2)) + freq_stats = self._freq_stats + + while not self._stop_event.is_set(): + loop_start = self._now() + loop_count += 1 + + if loop_count > 1: + freq_stats = self._update_frequency_stats(loop_start, prev_loop_start, freq_stats) + prev_loop_start = loop_start + + # 1) Poll external commands + self._poll_signals(loop_start) + + # 2) Manage the primary move queue (start new move, end finished move, breathing) + self._update_primary_motion(loop_start) + + # 3) Build the primary full-body pose for this tick + head, antennas, body_yaw = self._get_primary_pose(loop_start) + + # 4) Apply listening antenna freeze or blend-back + antennas_cmd = self._calculate_blended_antennas(antennas) + + # 5) Single set_target call - the only control point + self._issue_control_command(head, antennas_cmd, body_yaw) + + # 6) Adaptive sleep to align to next tick, then publish shared state + sleep_time, freq_stats = self._schedule_next_tick(loop_start, freq_stats) + self._publish_shared_state() + self._record_frequency_snapshot(freq_stats) + + # 7) Periodic telemetry on loop frequency + self._maybe_log_frequency(loop_count, print_interval_loops, freq_stats) + + if sleep_time > 0: + time.sleep(sleep_time) + + logger.debug("Movement control loop stopped") diff --git a/src/peachy/openai_realtime.py b/src/peachy/openai_realtime.py new file mode 100644 index 0000000000000000000000000000000000000000..0ef8be31e622f4b1c9e9ab48f3a716bff5a2ad83 --- /dev/null +++ b/src/peachy/openai_realtime.py @@ -0,0 +1,1178 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +import json +import os +import random +import time +import uuid +import base64 +import asyncio +import logging +from collections import deque +from typing import Any, Final, Literal, cast + +import numpy as np +from openai import APIConnectionError, APITimeoutError, AsyncOpenAI +from pydantic import Field, BaseModel +from openai.types.realtime import ( + AudioTranscriptionParam, + RealtimeAudioConfigParam, + RealtimeToolsConfigParam, + RealtimeFunctionToolParam, + RealtimeAudioConfigInputParam, + RealtimeAudioConfigOutputParam, + RealtimeSessionCreateRequestParam, +) +from websockets.exceptions import ConnectionClosedError +from openai.resources.realtime.realtime import AsyncRealtimeConnection +from openai.types.realtime.realtime_audio_input_turn_detection_param import ServerVad + +from peachy.tools import core_tools +from peachy.config import config +from peachy.prompts import ( + get_session_voice, + get_session_instructions, + get_session_greeting_prompt, +) +from peachy.audio.resampling import pcm16_at_rate +from peachy.streaming import AdditionalOutputs +from peachy.tools.core_tools import ( + ToolSpec, + ToolDependencies, + get_tool_specs, +) +from peachy.conversation_handler import AudioFrame, ConversationHandler +from peachy.providers import redact_secrets +from peachy.tools.background_tool_manager import ( + ToolCallRoutine, + ToolNotification, + BackgroundToolManager, +) + + +logger = logging.getLogger(__name__) + +_RESPONSE_DONE_TIMEOUT: Final[float] = 30.0 +_RESPONSE_REJECTION_RETRY_DELAY: Final[float] = 0.5 +OPENAI_SAMPLE_RATE: Final[int] = 24_000 +OPENAI_DEFAULT_MODEL: Final[str] = "gpt-realtime" +OPENAI_DEFAULT_VOICE: Final[str] = "marin" +OPENAI_VOICES: Final[tuple[str, ...]] = ( + "alloy", + "ash", + "ballad", + "cedar", + "coral", + "echo", + "marin", + "sage", + "shimmer", + "verse", +) + +ErrorDisposition = Literal["terminal", "transient"] + + +class TerminalProviderError(RuntimeError): + """A provider failure that needs configuration or operator action.""" + + +class RetryExhaustedError(RuntimeError): + """Transient provider failures exhausted this activation's retry budget.""" + + +class RealtimeProtocolError(RuntimeError): + """A server-delivered Realtime error event that must end this session.""" + + def __init__( + self, + message: str, + *, + code: str = "", + status_code: int | None = None, + event_id: str | None = None, + ) -> None: + super().__init__(message) + self.code = code + self.status_code = status_code + self.event_id = event_id + + +def _protocol_error_from_event(event: object) -> RealtimeProtocolError: + payload = getattr(event, "error", None) + + def field(name: str) -> object: + if isinstance(payload, dict): + return payload.get(name) + return getattr(payload, name, None) + + code = str(field("code") or field("type") or "").lower() + message = str(field("message") or "OpenAI Realtime protocol error") + raw_status = field("status_code") + status = raw_status if isinstance(raw_status, int) else None + raw_event_id = field("event_id") + event_id = raw_event_id if isinstance(raw_event_id, str) else None + return RealtimeProtocolError( + message, + code=code, + status_code=status, + event_id=event_id, + ) + + +def _error_code(error: BaseException) -> str: + direct = getattr(error, "code", None) + if isinstance(direct, str): + return direct.lower() + body = getattr(error, "body", None) + if isinstance(body, dict): + code = body.get("code") + nested = body.get("error") + if not code and isinstance(nested, dict): + code = nested.get("code") + return str(code or "").lower() + return "" + + +def classify_openai_error(error: BaseException) -> ErrorDisposition: + """Classify an OpenAI activation failure without permitting retry storms.""" + code = _error_code(error) + status = getattr(error, "status_code", None) + if code in {"insufficient_quota", "invalid_api_key", "model_not_found"}: + return "terminal" + if code in {"rate_limit_exceeded", "server_error"}: + return "transient" + if status in {400, 401, 403, 404, 422}: + return "terminal" + if status == 429: + return "transient" + if isinstance( + error, + ( + APIConnectionError, + APITimeoutError, + ConnectionError, + TimeoutError, + ConnectionClosedError, + ), + ): + return "transient" + if isinstance(status, int) and status >= 500: + return "transient" + return "terminal" + + +class InputTranscriptChunksByItem(BaseModel): + """Current item_id and its accumulated deltas. Only one item at a time.""" + + item_id: str | None = None + deltas: list[str] = Field(default_factory=list) + + +def to_realtime_tools_config(tool_specs: list[ToolSpec]) -> RealtimeToolsConfigParam: + """Convert app tool specs to the OpenAI-compatible realtime session shape.""" + realtime_tools: RealtimeToolsConfigParam = [] + for spec in tool_specs: + realtime_tools.append( + RealtimeFunctionToolParam( + type="function", + name=spec["name"], + description=spec["description"], + parameters=spec["parameters"], + ) + ) + return realtime_tools + + +def _pcm24k() -> dict[str, object]: + return {"type": "audio/pcm", "rate": OPENAI_SAMPLE_RATE} + + +class OpenAIRealtimeHandler(ConversationHandler): + """Realtime stream handler for the direct OpenAI Realtime API.""" + + SAMPLE_RATE = OPENAI_SAMPLE_RATE + + def __init__( + self, + deps: ToolDependencies, + instance_path: str | None = None, + startup_voice: str | None = None, + ) -> None: + """Initialize the handler.""" + super().__init__() + + self.deps = deps + + self.client: AsyncOpenAI + self.connection: AsyncRealtimeConnection | None = None + self.output_queue: asyncio.Queue[AudioFrame | AdditionalOutputs] = asyncio.Queue() + + self.instance_path = instance_path + self._voice_override: str | None = self._normalize_startup_voice(startup_voice) + self.model = (os.getenv("OPENAI_REALTIME_MODEL") or OPENAI_DEFAULT_MODEL).strip() or OPENAI_DEFAULT_MODEL + + # Debouncing for partial transcripts + self.partial_transcript_task: asyncio.Task[None] | None = None + self.partial_debounce_delay = 0.5 # seconds + self.input_transcript_chunks_by_item = InputTranscriptChunksByItem() + + # Internal lifecycle flags + self._connected_event: asyncio.Event = asyncio.Event() + + # Background tool manager + self.tool_manager = BackgroundToolManager() + + # Response-in-progress guard: the Realtime API only allows one active + # response per conversation at a time. A dedicated worker task + # (_response_sender_loop) dequeues and sends one request at a time + self._pending_responses: asyncio.Queue[dict[str, Any]] = asyncio.Queue() + self._response_done_event: asyncio.Event = asyncio.Event() + self._response_done_event.set() + self._response_started_or_rejected_event: asyncio.Event = asyncio.Event() + self._last_response_rejected: bool = False + self._turn_user_done_at: float | None = None + self._turn_response_created_at: float | None = None + self._turn_first_audio_at: float | None = None + self._startup_greeting_sent = False + self._in_flight_tool_calls: set[str] = set() + self._tool_batch_needs_response = False + self._response_generation = 0 + self._active_response_id: str | None = None + self._active_response_generation: int | None = None + self._cancelled_response_ids: set[str] = set() + self._cancelled_generations: set[int] = set() + self._pending_cancel_event_ids: deque[str] = deque(maxlen=32) + self._restart_tasks: set[asyncio.Task[None]] = set() + self._restart_lock = asyncio.Lock() + self._shutdown_started = False + + @staticmethod + def _sanitize_tool_result_for_model(tool_name: str, tool_result: dict[str, Any]) -> dict[str, Any]: + """Remove bulky transport-only fields before echoing tool output back to the model.""" + if tool_name == "camera" and "b64_im" in tool_result: + sanitized = dict(tool_result) + sanitized.pop("b64_im", None) + sanitized["image_attached"] = True + return sanitized + return tool_result + + @staticmethod + def _event_response_id(event: object) -> str | None: + direct = getattr(event, "response_id", None) + if isinstance(direct, str) and direct: + return direct + response = getattr(event, "response", None) + nested = getattr(response, "id", None) + return nested if isinstance(nested, str) and nested else None + + def _event_belongs_to_cancelled_response(self, event: object) -> bool: + response_id = self._event_response_id(event) + if response_id is not None and response_id in self._cancelled_response_ids: + return True + generation = self._active_response_generation + if generation is None or generation not in self._cancelled_generations: + return False + # response.created.response.id is optional in SDK 2.28, while audio deltas + # carry response_id. Bind that first observed ID to the already-cancelled + # active generation and suppress it until response.done. + if response_id is not None: + self._cancelled_response_ids.add(response_id) + if self._active_response_id is None: + self._active_response_id = response_id + return True + + def _forget_cancel_event_id(self, event_id: str) -> None: + try: + self._pending_cancel_event_ids.remove(event_id) + except ValueError: + pass + + def _finish_response(self, event: object) -> None: + response_id = self._event_response_id(event) + if response_id is not None: + self._cancelled_response_ids.discard(response_id) + if self._active_response_generation is not None: + self._cancelled_generations.discard(self._active_response_generation) + self._active_response_id = None + self._active_response_generation = None + + def _restart_task_done(self, task: asyncio.Task[None]) -> None: + """Retrieve and sanitize background restart failures.""" + self._restart_tasks.discard(task) + if task.cancelled(): + return + try: + task.result() + except Exception as exc: + logger.warning("Realtime session restart failed: %s", redact_secrets(exc)) + + async def _cancel_restart_tasks(self) -> None: + """Cancel and retrieve retained restart tasks during handler shutdown.""" + restart_tasks = tuple(self._restart_tasks) + for task in restart_tasks: + task.cancel() + if restart_tasks: + await asyncio.gather(*restart_tasks, return_exceptions=True) + self._restart_tasks.clear() + + async def _cancel_active_response_for_barge_in(self) -> None: + response_id = self._active_response_id + generation = self._active_response_generation + if response_id is not None: + self._cancelled_response_ids.add(response_id) + if generation is not None: + self._cancelled_generations.add(generation) + + if self.connection is not None and (response_id is not None or generation is not None): + cancel_event_id = f"peachy-cancel-{uuid.uuid4().hex}" + self._pending_cancel_event_ids.append(cancel_event_id) + try: + if response_id is None: + await self.connection.response.cancel(event_id=cancel_event_id) + else: + await self.connection.response.cancel( + event_id=cancel_event_id, + response_id=response_id, + ) + except Exception as exc: + self._forget_cancel_event_id(cancel_event_id) + logger.debug("Response cancellation raced with completion (%s)", type(exc).__name__) + + if self._clear_queue is not None: + self._clear_queue() + else: + while not self.output_queue.empty(): + try: + self.output_queue.get_nowait() + except asyncio.QueueEmpty: + break + + def _normalize_startup_voice(self, voice: str | None) -> str | None: + """Return a valid persisted startup voice, or None.""" + return self._resolve_backend_voice(voice, source="persisted startup voice") + + async def _wait_for_response_done_before_tool_result(self) -> bool: + """Return whether the function-call response finished before sending tool output.""" + if self._response_done_event.is_set(): + return True + + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + return True + except asyncio.TimeoutError: + return False + + def _resolve_backend_voice( + self, + voice: str | None, + *, + source: str, + fallback: str | None = None, + ) -> str | None: + """Return a backend-supported voice, optionally falling back when unsupported.""" + available_voices = OPENAI_VOICES + voice_value = (voice or "").strip() + if not voice_value: + return fallback + + voice_by_lowercase = {candidate.lower(): candidate for candidate in available_voices} + normalized_voice = voice_by_lowercase.get(voice_value.lower()) + if normalized_voice is not None: + return normalized_voice + + if voice: + logger.warning( + "Ignoring unsupported %s %r; expected one of %s", + source, + voice, + available_voices, + ) + return fallback + + def _get_session_config(self, tool_specs: list[ToolSpec]) -> RealtimeSessionCreateRequestParam: + """Return the direct OpenAI realtime session config.""" + return RealtimeSessionCreateRequestParam( + type="realtime", + instructions=get_session_instructions(self.instance_path), + audio=RealtimeAudioConfigParam( + input=RealtimeAudioConfigInputParam( + format=_pcm24k(), # type: ignore[typeddict-item] + transcription=AudioTranscriptionParam( + model="gpt-4o-transcribe", + language=config.REALTIME_TRANSCRIPTION_LANGUAGE, + ), + turn_detection=ServerVad(type="server_vad", interrupt_response=True), + ), + output=RealtimeAudioConfigOutputParam( + format=_pcm24k(), # type: ignore[typeddict-item] + voice=self.get_current_voice(), + ), + ), + tools=to_realtime_tools_config([spec for spec in tool_specs if spec["name"] != "camera"]), + tool_choice="auto", + ) + + def _is_connected(self) -> bool: + """Return whether the realtime connection is open.""" + return self.connection is not None + + def _idle_behavior_ready(self) -> bool: + """Hold idle behavior while a model response is still active.""" + return self._response_done_event.is_set() + + async def _cancel_partial_transcript_task(self) -> None: + if self.partial_transcript_task and not self.partial_transcript_task.done(): + self.partial_transcript_task.cancel() + try: + await self.partial_transcript_task + except asyncio.CancelledError: + pass + + async def change_voice(self, voice: str) -> str: + """Change only the voice, updating the active session when possible.""" + resolved_voice = ( + self._resolve_backend_voice(voice, source="requested voice", fallback=OPENAI_DEFAULT_VOICE) + or OPENAI_DEFAULT_VOICE + ) + self._voice_override = resolved_voice + if self.connection is not None: + try: + await self.connection.session.update( + session=RealtimeSessionCreateRequestParam( + type="realtime", + audio=RealtimeAudioConfigParam( + output=RealtimeAudioConfigOutputParam( + voice=resolved_voice, + ), + ), + ), + ) + return f"Voice changed to {resolved_voice}." + except Exception as e: + logger.warning("Failed to update live session for voice change: %s", redact_secrets(e)) + return "Voice change failed. Will take effect on next connection." + return "Voice changed. Will take effect on next connection." + + def get_current_voice(self) -> str: + """Return the voice currently selected for this handler.""" + voice = self._voice_override or get_session_voice(default=OPENAI_DEFAULT_VOICE) + return ( + self._resolve_backend_voice(voice, source="session voice", fallback=OPENAI_DEFAULT_VOICE) + or OPENAI_DEFAULT_VOICE + ) + + async def apply_personality(self, profile: str | None) -> str: + """Apply a new personality (profile) at runtime if possible. + + - Updates the global config's selected profile for subsequent calls. + - If a realtime connection is active, sends a session.update with the + freshly resolved instructions so the change takes effect immediately. + + Returns a short status message for UI feedback. + """ + try: + # Update the in-process config value and env + from peachy.config import config as _config + from peachy.config import set_custom_profile + + set_custom_profile(profile) + logger.info( + "Set custom profile to %r (config=%r)", profile, getattr(_config, "REACHY_MINI_CUSTOM_PROFILE", None) + ) + + try: + instructions = get_session_instructions(self.instance_path) + voice = self.get_current_voice() + except Exception as e: + safe_error = redact_secrets(e) + logger.error("Failed to resolve personality content: %s", safe_error) + return f"Failed to apply personality: {safe_error}" + + # Rebuild the tool registry + core_tools.initialize_tools(force=True) + + # Attempt a live update first, then force a full restart to ensure it sticks + if self.connection is not None: + try: + await self.connection.session.update( + session=RealtimeSessionCreateRequestParam( + type="realtime", + instructions=instructions, + audio=RealtimeAudioConfigParam( + output=RealtimeAudioConfigOutputParam( + voice=voice, + ), + ), + ), + ) + logger.info("Applied personality via live update: %s", profile or "built-in default") + except Exception as e: + logger.warning("Live update failed; will restart session: %s", redact_secrets(e)) + + # Force a real restart to guarantee the new instructions/voice + try: + await self._restart_session() + return "Applied personality and restarted realtime session." + except Exception as e: + logger.warning("Failed to restart session after apply: %s", redact_secrets(e)) + return "Applied personality. Will take effect on next connection." + else: + logger.info( + "Applied personality recorded: %s (no live connection; will apply on next session)", + profile or "built-in default", + ) + return "Applied personality. Will take effect on next connection." + except Exception as e: + safe_error = redact_secrets(e) + logger.error("Error applying personality '%s': %s", profile, safe_error) + return f"Failed to apply personality: {safe_error}" + + async def _emit_debounced_partial(self, transcript: str, item_id: str, sequence_counter: int) -> None: + """Emit partial transcript after debounce delay.""" + try: + await asyncio.sleep(self.partial_debounce_delay) + + input_transcript = self.input_transcript_chunks_by_item + if input_transcript.item_id == item_id and len(input_transcript.deltas) - 1 == sequence_counter: + await self.output_queue.put(AdditionalOutputs({"role": "user_partial", "content": transcript})) + logger.debug(f"Debounced partial emitted: {transcript}") + except asyncio.CancelledError: + logger.debug("Debounced partial cancelled") + raise + + def _record_partial_transcript_delta( + self, + input_transcript: InputTranscriptChunksByItem, + item_id: str, + delta: str, + ) -> None: + """Record an OpenAI partial transcript snapshot.""" + input_transcript.item_id = item_id + input_transcript.deltas = [delta] + + async def start_up(self) -> None: + """Run one bounded OpenAI activation.""" + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + try: + self.client = await self._build_realtime_client() + await self._run_realtime_session() + return + except asyncio.CancelledError: + raise + except Exception as exc: + message = redact_secrets(exc) + if classify_openai_error(exc) == "terminal": + logger.warning("OpenAI realtime activation stopped: %s", message) + raise TerminalProviderError(message) from exc + if attempt == max_attempts: + logger.warning( + "OpenAI realtime activation exhausted %d attempts: %s", + max_attempts, + message, + ) + raise RetryExhaustedError( + f"OpenAI realtime failed after {max_attempts} attempts: {message}" + ) from exc + delay = float(2 ** (attempt - 1)) + random.uniform(0.0, 0.5) + logger.warning( + "OpenAI realtime transient failure (%d/%d): %s; retrying in %.1fs", + attempt, + max_attempts, + message, + delay, + ) + await asyncio.sleep(delay) + finally: + self.connection = None + self._connected_event.clear() + + async def _restart_session(self) -> None: + """Force-close the current session and start a fresh one in background. + + Does not block the caller while the new session is establishing. + """ + try: + async with self._restart_lock: + if self._shutdown_started: + return + + if self.connection is not None: + try: + await self.connection.close() + except Exception: + pass + finally: + self.connection = None + + # Ensure we have a client (start_up must have run once) + if getattr(self, "client", None) is None: + logger.warning("Cannot restart: realtime client not initialized yet.") + return + + # Build before registering the session task, then re-check the + # shutdown gate because client construction may have suspended. + try: + self._connected_event.clear() + except Exception: + pass + client = await self._build_realtime_client() + if self._shutdown_started: + return + self.client = client + restart_task = asyncio.create_task( + self._run_realtime_session(), + name="realtime-session-restart", + ) + self._restart_tasks.add(restart_task) + restart_task.add_done_callback(self._restart_task_done) + try: + await asyncio.wait_for(self._connected_event.wait(), timeout=5.0) + if not self._shutdown_started: + logger.info("Realtime session restarted and connected.") + except asyncio.TimeoutError: + if not self._shutdown_started: + logger.warning("Realtime session restart timed out; continuing in background.") + except asyncio.CancelledError: + raise + except Exception as e: + logger.warning("_restart_session failed: %s", redact_secrets(e)) + + async def _safe_response_create(self, **kwargs: Any) -> None: + """Enqueue a response.create() kwargs for the sender worker _response_sender_loop(). + + This method never blocks the caller. + """ + await self._pending_responses.put(kwargs) + + async def _send_startup_greeting_prompt(self) -> None: + """Prompt the model to open the conversation once the session is ready.""" + if self._startup_greeting_sent or not self.connection: + return + + greeting_prompt = get_session_greeting_prompt().strip() + if not greeting_prompt: + self._startup_greeting_sent = True + return + + try: + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": greeting_prompt, + }, + ], + }, + ) + self._startup_greeting_sent = True + self._mark_activity("startup_greeting_prompt") + await self._safe_response_create() + logger.info("Queued startup greeting prompt") + except Exception as e: + logger.warning("Failed to queue startup greeting prompt: %s", redact_secrets(e)) + + async def _response_sender_loop(self) -> None: + """Dedicated worker that sends ``response.create()`` calls serially. + + This logic was designed to comply with the response.create() docstring specification for event ordering: + https://github.com/openai/openai-python/blob/3e0c05b84a2056870abf3bd6a5e7849020209cc3/src/openai/resources/realtime/realtime.py#L649C1-L651C30 + + For each queued request the worker: + 1. Waits until no response is active (_response_done_event). + 2. Sends response.create(). + 3. Waits until the receiver observes response.created or a rejection. + 4. Waits for the response cycle to complete (response.done). + 5. If the server rejected with active_response, retries from step 1. + """ + while self.connection: + try: + kwargs = await self._pending_responses.get() + except asyncio.CancelledError: + return + + # Parallel tool calls enqueue duplicate empty requests; coalesce to one. + while not kwargs and not self._pending_responses.empty(): + try: + self._pending_responses.get_nowait() + except asyncio.QueueEmpty: + break + + sent = False + max_retries = 5 + attempts = 0 + while not sent and self.connection and attempts < max_retries: + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for previous response to finish; forcing ahead") + self._response_done_event.set() + + if not self.connection: + break + + self._last_response_rejected = False + self._response_started_or_rejected_event.clear() + try: + await self.connection.response.create(**kwargs) + except Exception as e: + logger.debug("_response_sender_loop: send failed: %s", redact_secrets(e)) + self._response_done_event.set() + break + + try: + await asyncio.wait_for( + self._response_started_or_rejected_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for response.created or response rejection") + + # Check if the receiver loop observed an asynchronous rejection. + if self._last_response_rejected: + attempts += 1 + if attempts >= max_retries: + logger.debug("response.create rejected %d times; giving up", attempts) + break + logger.debug("response.create was rejected; retrying (%d/%d)", attempts, max_retries) + await asyncio.sleep(_RESPONSE_REJECTION_RETRY_DELAY) + continue + + try: + await asyncio.wait_for( + self._response_done_event.wait(), + timeout=_RESPONSE_DONE_TIMEOUT, + ) + except asyncio.TimeoutError: + logger.debug("Timed out waiting for response.done; assuming response completed") + self._response_done_event.set() + break + + sent = True + + async def _handle_tool_result(self, completed_tool: ToolNotification) -> None: + """Process the result of a tool call.""" + if completed_tool.error is not None: + logger.error( + "Tool '%s' (id=%s) failed with error: %s", + completed_tool.tool_name, + completed_tool.id, + completed_tool.error, + ) + tool_result = {"error": completed_tool.error} + tool_result_for_model = tool_result + elif completed_tool.result is not None: + tool_result = completed_tool.result + tool_result_for_model = ( + self._sanitize_tool_result_for_model(completed_tool.tool_name, tool_result) + if isinstance(tool_result, dict) + else tool_result + ) + logger.info( + "Tool '%s' (id=%s) executed successfully.", + completed_tool.tool_name, + completed_tool.id, + ) + logger.debug("Tool '%s' model-visible result: %s", completed_tool.tool_name, tool_result_for_model) + else: + logger.warning( + "Tool '%s' (id=%s) returned no result and no error", completed_tool.tool_name, completed_tool.id + ) + tool_result = {"error": "No result returned from tool execution"} + tool_result_for_model = tool_result + + # Connection may have closed while tool was running + if not self.connection: + logger.warning( + "Connection closed during tool '%s' (id=%s) execution; cannot send result back", + completed_tool.tool_name, + completed_tool.id, + ) + return + + try: + send_result_to_model = not completed_tool.is_idle_tool_call + if send_result_to_model: + self._mark_activity("tool_result_ready") + model_result_submitted = False + if send_result_to_model and isinstance(completed_tool.id, str): + if not await self._wait_for_response_done_before_tool_result(): + send_result_to_model = False + if not send_result_to_model: + logger.warning( + "Dropping realtime model result for tool '%s' (id=%s) because response.done was not observed", + completed_tool.tool_name, + completed_tool.id, + ) + elif not self.connection: + logger.warning( + "Connection closed before sending tool '%s' (id=%s) result back", + completed_tool.tool_name, + completed_tool.id, + ) + return + else: + await self.connection.conversation.item.create( + item={ + "type": "function_call_output", + "call_id": completed_tool.id, + "output": json.dumps(tool_result_for_model), + }, + ) + model_result_submitted = True + + await self.output_queue.put( + AdditionalOutputs( + { + "role": "assistant", + "content": json.dumps(tool_result_for_model), + }, + ), + ) + + if model_result_submitted and completed_tool.tool_name == "camera" and "b64_im" in tool_result: + # use raw base64, don't json.dumps (which adds quotes) + b64_im = tool_result["b64_im"] + if not isinstance(b64_im, str): + logger.warning("Unexpected type for b64_im: %s", type(b64_im)) + b64_im = str(b64_im) + image_width = tool_result.get("image_width") + image_height = tool_result.get("image_height") + jpeg_bytes_value = tool_result.get("jpeg_bytes") + jpeg_bytes = jpeg_bytes_value if isinstance(jpeg_bytes_value, int) else (len(b64_im) * 3) // 4 + await self.connection.conversation.item.create( + item={ + "type": "message", + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": f"data:image/jpeg;base64,{b64_im}", + }, + ], + }, + ) + if isinstance(image_width, int) and isinstance(image_height, int): + logger.info( + "Added camera image to conversation frame=%sx%s jpeg_bytes=%s", + image_width, + image_height, + jpeg_bytes, + ) + else: + logger.info( + "Added camera image to conversation jpeg_bytes=%s", + jpeg_bytes, + ) + + if isinstance(completed_tool.id, str): + self._in_flight_tool_calls.discard(completed_tool.id) + + tool = core_tools.ALL_TOOLS.get(completed_tool.tool_name) + # Always surface errors, skip the spoken follow-up for tools that opt out. + if model_result_submitted and (completed_tool.error is not None or tool is None or tool.needs_response): + self._tool_batch_needs_response = True + + # Parallel tool calls in one turn: respond once every result is in, not per tool. + if self._tool_batch_needs_response and not self._in_flight_tool_calls: + self._tool_batch_needs_response = False + await self._safe_response_create() + + except ConnectionClosedError: + logger.warning("Connection closed while sending tool result") + self.connection = None + self._response_done_event.set() + + async def _run_realtime_session(self) -> None: + """Establish and manage a single direct OpenAI realtime session.""" + tool_specs = [spec for spec in get_tool_specs() if spec["name"] != "camera"] + async with self.client.realtime.connect(model=self.model) as conn: + try: + await conn.session.update(session=self._get_session_config(tool_specs)) + except Exception as exc: + logger.error("OpenAI realtime session.update failed: %s", redact_secrets(exc)) + raise + self.connection = conn + self._connected_event.set() + response_sender_task: asyncio.Task[None] | None = None + try: + self.tool_manager.start_up(tool_callbacks=[self._handle_tool_result]) + response_sender_task = asyncio.create_task(self._response_sender_loop(), name="response-sender") + await self._send_startup_greeting_prompt() + async for event in conn: + logger.debug("Realtime event: %s", event.type) + await self._handle_event(event) + finally: + if response_sender_task is not None: + response_sender_task.cancel() + try: + await response_sender_task + except asyncio.CancelledError: + pass + await self.tool_manager.shutdown() + self.connection = None + self._connected_event.clear() + + async def _handle_event(self, event: object) -> None: + """Handle one OpenAI Realtime event.""" + event_data = cast(Any, event) + if event_data.type == "input_audio_buffer.speech_started": + self._mark_activity("user_speech_started") + self._turn_user_done_at = None + self._turn_response_created_at = None + self._turn_first_audio_at = None + await self._cancel_active_response_for_barge_in() + self.deps.movement_manager.set_listening(True) + logger.debug("User speech started") + + if event_data.type == "input_audio_buffer.speech_stopped": + self._mark_activity("user_speech_stopped") + self.deps.movement_manager.set_listening(False) + logger.debug("User speech stopped - server will auto-commit with VAD") + + if event_data.type == "response.output_audio.done": + self.deps.movement_manager.set_speaking(False) + logger.debug("response completed") + + if event_data.type == "response.output_text.delta": + logger.debug("response text delta") + + if event_data.type == "response.output_text.done": + logger.debug("response text done: %s", event_data.text) + + if event_data.type == "response.created": + self._mark_activity("response_created") + self._response_generation += 1 + self._active_response_generation = self._response_generation + self._active_response_id = self._event_response_id(event) + self.deps.movement_manager.set_speaking(True) + self._response_done_event.clear() + self._response_started_or_rejected_event.set() + if self._turn_user_done_at is not None and self._turn_response_created_at is None: + self._turn_response_created_at = time.perf_counter() + delta_ms = (self._turn_response_created_at - self._turn_user_done_at) * 1000 + logger.info("Turn latency: response.created %.0f ms after user transcript", delta_ms) + logger.debug("Response created (active)") + + if event_data.type == "response.done": + # Doesn't mean the audio is done playing + # Resume tracking for responses that emit no audio (text-only / tool-only). + self.deps.movement_manager.set_speaking(False) + self._response_done_event.set() + self._response_started_or_rejected_event.set() + self._finish_response(event) + logger.debug("Response done") + + if event_data.type == "conversation.item.input_audio_transcription.delta": + self._mark_activity("user_transcription_delta") + logger.debug(f"User partial transcript: {event_data.delta}") + + item_id = event_data.item_id + delta = event_data.delta or "" + + input_transcript = self.input_transcript_chunks_by_item + self._record_partial_transcript_delta(input_transcript, item_id, delta) + + current_partial = "".join(input_transcript.deltas) + sequence_counter = len(input_transcript.deltas) - 1 + + await self._cancel_partial_transcript_task() + + # Start new debounce timer with the last delta + self.partial_transcript_task = asyncio.create_task( + self._emit_debounced_partial(current_partial, item_id, sequence_counter) + ) + + # Handle completed transcription (user finished speaking) + if event_data.type == "conversation.item.input_audio_transcription.completed": + self._mark_activity("user_transcription_completed") + raw_transcript = event_data.transcript or "" + transcript = raw_transcript.strip() + logger.debug("User transcript: %s", raw_transcript) + self.deps.movement_manager.set_listening(False) + + await self._cancel_partial_transcript_task() + + if not transcript: + logger.debug("Ignoring empty user transcript") + return + + self._turn_user_done_at = time.perf_counter() + self._turn_response_created_at = None + self._turn_first_audio_at = None + self._in_flight_tool_calls.clear() + self._tool_batch_needs_response = False + + await self.output_queue.put(AdditionalOutputs({"role": "user", "content": transcript})) + + # Handle assistant transcription + if event_data.type == "response.output_audio_transcript.done": + self._mark_activity("assistant_transcript_done") + logger.debug(f"Assistant transcript: {event_data.transcript}") + await self.output_queue.put( + AdditionalOutputs({"role": "assistant", "content": event_data.transcript}) + ) + + # Handle audio delta + if event_data.type == "response.output_audio.delta": + if self._event_belongs_to_cancelled_response(event): + logger.debug("Discarding audio delta from cancelled response") + return + decoded_pcm = np.frombuffer(base64.b64decode(event_data.delta), dtype=" None: + """Append robot microphone audio as OpenAI's required mono 24 kHz PCM16.""" + if self.connection is None: + return + source_rate, source = frame + if source.size == 0: + return + pcm = pcm16_at_rate(source, source_rate, self.SAMPLE_RATE) + encoded = base64.b64encode(pcm.astype(" None: + """Shutdown the handler.""" + self._shutdown_started = True + self._connected_event.set() + + # Wait for any restart still constructing its client to observe the + # shutdown gate before taking the complete registered-task snapshot. + async with self._restart_lock: + pass + + # Unblock the response sender worker so it can exit + self._response_done_event.set() + self._pending_cancel_event_ids.clear() + + await self._cancel_restart_tasks() + + # Stop background tool manager tasks (listener + cleanup) + await self.tool_manager.shutdown() + + await self._cancel_partial_transcript_task() + + if self.connection: + try: + await self.connection.close() + except ConnectionClosedError as e: + logger.debug("Connection already closed during shutdown: %s", redact_secrets(e)) + except Exception as e: + logger.debug("connection.close() ignored: %s", redact_secrets(e)) + finally: + self.connection = None + self.connection = None + + # Clear any remaining items in the output queue + while not self.output_queue.empty(): + try: + self.output_queue.get_nowait() + except asyncio.QueueEmpty: + break + + async def get_available_voices(self) -> list[str]: + """Return the available OpenAI voices.""" + return list(OPENAI_VOICES) + + async def _build_realtime_client(self) -> AsyncOpenAI: + """Build the direct OpenAI realtime client.""" + api_key = (os.getenv("OPENAI_API_KEY") or "").strip() + if not api_key: + raise RuntimeError("OPENAI_API_KEY is required for the OpenAI realtime provider") + self.model = (os.getenv("OPENAI_REALTIME_MODEL") or OPENAI_DEFAULT_MODEL).strip() or OPENAI_DEFAULT_MODEL + return AsyncOpenAI(api_key=api_key) diff --git a/src/peachy/personality.py b/src/peachy/personality.py new file mode 100644 index 0000000000000000000000000000000000000000..e2248ad6d93f746ead3cf661e2753d39a8a26a98 --- /dev/null +++ b/src/peachy/personality.py @@ -0,0 +1,157 @@ +"""Personality (profile) data layer. + +Provides functions to list, read, and write personality profiles stored +on disk. No HTTP or framework dependencies — importable anywhere. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +from typing import List +from pathlib import Path + +from .config import DEFAULT_PROFILES_DIRECTORY, USER_PERSONALITIES_DIRNAME, config, get_default_voice +from .tools.tool_constants import SystemTool + + +DEFAULT_OPTION = "(built-in default)" + +# Dev-only profiles, hidden from the UI, but still loadable via REACHY_MINI_CUSTOM_PROFILE +UNLISTED_PROFILES = {"tedai"} + + +def _tools_dir() -> Path: + return Path(__file__).parent / "tools" + + +def _sanitize_name(name: str) -> str: + import re + + s = name.strip() + s = re.sub(r"\s+", "_", s) + s = re.sub(r"[^a-zA-Z0-9_-]", "", s) + return s + + +def list_personalities() -> List[str]: + """List available personality profile names.""" + names: List[str] = [] + try: + builtin_root = config.PROFILES_DIRECTORY + if builtin_root.exists(): + for p in sorted(builtin_root.iterdir()): + if p.name == USER_PERSONALITIES_DIRNAME or p.name in UNLISTED_PROFILES: + continue + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(p.name) + udir = config.user_personalities_root() + if udir.exists(): + for p in sorted(udir.iterdir()): + if p.is_dir() and (p / "instructions.txt").exists(): + names.append(f"{USER_PERSONALITIES_DIRNAME}/{p.name}") + except Exception: + pass + return names + + +def resolve_profile_dir(selection: str) -> Path: + """Resolve the directory path for the given profile selection.""" + return config.resolve_profile_dir(selection) + + +def read_instructions_for(name: str) -> str: + """Read the instructions.txt content for the given profile name.""" + try: + if name == DEFAULT_OPTION: + target = DEFAULT_PROFILES_DIRECTORY / "default" / "instructions.txt" + return target.read_text(encoding="utf-8").strip() if target.exists() else "" + target = resolve_profile_dir(name) / "instructions.txt" + return target.read_text(encoding="utf-8").strip() if target.exists() else "" + except Exception as e: + return f"Could not load instructions: {e}" + + +def read_tools_for(name: str) -> str: + """Read the tools.txt content for the given profile name.""" + try: + profile_name = "default" if name == DEFAULT_OPTION else name + target = resolve_profile_dir(profile_name) / "tools.txt" + return target.read_text(encoding="utf-8") if target.exists() else "" + except Exception: + return "" + + +def read_greeting_for(name: str) -> str: + """Read the greeting.txt content for the given profile name.""" + try: + profile_name = "default" if name == DEFAULT_OPTION else name + target = resolve_profile_dir(profile_name) / "greeting.txt" + if target.exists(): + greeting = target.read_text(encoding="utf-8").strip() + if greeting: + return greeting + return "" + except Exception: + return "" + + +def available_tools_for(selected: str) -> List[str]: + """List available tool modules for the given profile selection.""" + shared: List[str] = [] + try: + for py in _tools_dir().glob("*.py"): + if py.stem in {"__init__", "core_tools", "background_tool_manager", "tool_constants"} or py.stem in { + t.value for t in SystemTool + }: + continue + shared.append(py.stem) + except Exception: + pass + local: List[str] = [] + try: + if selected != DEFAULT_OPTION: + for py in resolve_profile_dir(selected).glob("*.py"): + local.append(py.stem) + except Exception: + pass + return sorted(set(shared + local)) + + +def delete_personality(name: str) -> bool: + """Delete a user-created personality directory; return whether it existed. + + Refuses anything outside the user personalities root, so built-in profiles + can never be removed through the UI. + """ + import shutil + + target = resolve_profile_dir(name).resolve() + user_root = config.user_personalities_root().resolve() + if user_root not in target.parents: + return False + if target.is_dir(): + shutil.rmtree(target) + return True + return False + + +def _write_profile( + sanitized_name: str, + instructions: str, + tools_text: str, + voice: str | None = None, + greeting: str | None = None, +) -> None: + default_voice = get_default_voice() + target_dir = config.user_personalities_root() / sanitized_name + target_dir.mkdir(parents=True, exist_ok=True) + (target_dir / "instructions.txt").write_text(instructions.strip() + "\n", encoding="utf-8") + (target_dir / "tools.txt").write_text((tools_text or "").strip() + "\n", encoding="utf-8") + (target_dir / "voice.txt").write_text((voice or default_voice).strip() + "\n", encoding="utf-8") + if greeting is not None: + greeting_file = target_dir / "greeting.txt" + greeting_text = greeting.strip() + if greeting_text: + greeting_file.write_text(greeting_text + "\n", encoding="utf-8") + elif greeting_file.exists(): + greeting_file.unlink() diff --git a/src/peachy/personality_routes.py b/src/peachy/personality_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..91ed45273c1fe0d619383ad2572b0967185e0c09 --- /dev/null +++ b/src/peachy/personality_routes.py @@ -0,0 +1,313 @@ +"""FastAPI routes for personality and voice management. + +Exposes REST endpoints on the provided FastAPI app. Backend actions +(apply personality, fetch voices) are scheduled onto the running +LocalStream asyncio loop via the supplied get_loop callable. +""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import asyncio +import logging +from typing import Any, Callable, Optional, Awaitable + +from fastapi import Query, FastAPI, Request +from pydantic import BaseModel + +from .config import ( + LOCKED_PROFILE, + config, + get_default_voice, + get_available_voices, +) +from .personality import ( + DEFAULT_OPTION, + _sanitize_name, + _write_profile, + read_tools_for, + read_greeting_for, + delete_personality, + list_personalities, + available_tools_for, + resolve_profile_dir, + read_instructions_for, +) +from .conversation_handler import ConversationHandler + + +logger = logging.getLogger(__name__) + + +class ApplyPayload(BaseModel): + """Body of the apply-personality endpoint. + + Module-level: under postponed annotations, FastAPI can't resolve a + function-local model and silently treats it as a query param. + """ + + name: str + persist: bool = False + + +def mount_personality_routes( + app: FastAPI, + handler: ConversationHandler, + get_loop: Callable[[], asyncio.AbstractEventLoop | None], + *, + persist_personality: Callable[[Optional[str], Optional[str]], None] | None = None, + get_persisted_personality: Callable[[], Optional[str]] | None = None, + apply_personality: Callable[[Optional[str]], Awaitable[str]] | None = None, + get_voices: Callable[[], Awaitable[list[str]]] | None = None, + get_current_voice: Callable[[], str] | None = None, + change_voice: Callable[[str], Awaitable[str]] | None = None, + api_prefix: str | None = None, +) -> None: + """Register personality management endpoints on a FastAPI app.""" + from fastapi.responses import JSONResponse + + api_prefix = (api_prefix or "").rstrip("/") + + def _configured_startup_choice() -> Any: + """Return the startup personality configured when routes mount.""" + try: + if get_persisted_personality is not None: + stored = get_persisted_personality() + if stored: + return stored + env_val = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None) + if env_val: + return env_val + except Exception as e: + logger.warning("Failed to read configured startup personality: %s", e) + return DEFAULT_OPTION + + startup_choice = _configured_startup_choice() + + def _startup_choice() -> Any: + """Return the persisted startup personality or default.""" + try: + if get_persisted_personality is not None: + stored = get_persisted_personality() + if stored: + return stored + except Exception as e: + logger.warning("Failed to read persisted startup personality: %s", e) + return startup_choice + + def _set_startup_choice(selected_name: str) -> None: + nonlocal startup_choice + startup_choice = DEFAULT_OPTION if selected_name == DEFAULT_OPTION else selected_name + + def _current_choice() -> str: + try: + cur = getattr(config, "REACHY_MINI_CUSTOM_PROFILE", None) + return cur or DEFAULT_OPTION + except Exception: + return DEFAULT_OPTION + + def _voice_override() -> Optional[str]: + current_voice_callback = get_current_voice or getattr(handler, "get_current_voice", None) + return current_voice_callback() if callable(current_voice_callback) else None + + @app.get(f"{api_prefix}/personalities") + def _list() -> dict: # type: ignore + choices = [DEFAULT_OPTION, *list_personalities()] + return { + "choices": choices, + "current": _current_choice(), + "startup": _startup_choice(), + "locked": LOCKED_PROFILE is not None, + "locked_to": LOCKED_PROFILE, + } + + @app.get(f"{api_prefix}/personalities/load") + def _load(name: str) -> dict: # type: ignore + instr = read_instructions_for(name) + tools_txt = read_tools_for(name) + greeting = read_greeting_for(name) + voice = get_default_voice() + uses_default_voice = True + if name != DEFAULT_OPTION: + pdir = resolve_profile_dir(name) + vf = pdir / "voice.txt" + if vf.exists(): + v = vf.read_text(encoding="utf-8").strip() + voice = v or get_default_voice() + uses_default_voice = not bool(v) + avail = available_tools_for(name) + enabled = [ln.strip() for ln in tools_txt.splitlines() if ln.strip() and not ln.strip().startswith("#")] + return { + "instructions": instr, + "greeting": greeting, + "tools_text": tools_txt, + "voice": voice, + "uses_default_voice": uses_default_voice, + "available_tools": avail, + "enabled_tools": enabled, + } + + @app.post(f"{api_prefix}/personalities/save") + async def _save(request: Request) -> dict: # type: ignore + # Accept raw JSON only to avoid validation-related 422s + try: + raw = await request.json() + except Exception: + raw = {} + name = str(raw.get("name", "")) + instructions = str(raw.get("instructions", "")) + greeting = str(raw["greeting"]) if raw.get("greeting") is not None else None + tools_text = str(raw.get("tools_text", "")) + voice = str(raw.get("voice", get_default_voice())) if raw.get("voice") is not None else get_default_voice() + + sanitized_name = _sanitize_name(name) + if not sanitized_name: + return JSONResponse({"ok": False, "error": "invalid_name"}, status_code=400) # type: ignore + try: + logger.info( + "save: name=%r voice=%r instr_len=%d greeting_len=%d tools_len=%d", + sanitized_name, + voice, + len(instructions), + len(greeting or ""), + len(tools_text), + ) + _write_profile( + sanitized_name, + instructions, + tools_text, + voice or get_default_voice(), + greeting, + ) + value = f"user_personalities/{sanitized_name}" + choices = [DEFAULT_OPTION, *list_personalities()] + return {"ok": True, "value": value, "choices": choices} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + @app.delete(f"{api_prefix}/personalities") + def _delete(name: str) -> dict: # type: ignore + """Delete a user-created personality (name is the full selection string).""" + if name in (_current_choice(), _startup_choice()): + # Deleting the active/startup profile would break get_session_instructions() at next startup. + return JSONResponse( + {"ok": False, "error": "profile_in_use", "choices": [DEFAULT_OPTION, *list_personalities()]}, + status_code=409, + ) # type: ignore + deleted = delete_personality(name) + if not deleted: + # Built-in profile, outside the user root, or already gone; nothing was removed. + return JSONResponse( + {"ok": False, "error": "not_deletable", "choices": [DEFAULT_OPTION, *list_personalities()]}, + status_code=404, + ) # type: ignore + return {"ok": True, "choices": [DEFAULT_OPTION, *list_personalities()]} + + @app.post(f"{api_prefix}/personalities/apply") + async def _apply(payload: ApplyPayload) -> dict: # type: ignore + if LOCKED_PROFILE is not None: + return JSONResponse( + {"ok": False, "error": "profile_locked", "locked_to": LOCKED_PROFILE}, + status_code=403, + ) # type: ignore + selected_name = payload.name or DEFAULT_OPTION + persist = bool(payload.persist) + persisted_choice = _startup_choice() + + if selected_name == _current_choice(): + if persist and persist_personality is not None: + try: + voice_override = _voice_override() + persist_personality(None if selected_name == DEFAULT_OPTION else selected_name, voice_override) + _set_startup_choice(selected_name) + persisted_choice = _startup_choice() + except Exception as e: + logger.warning("Failed to persist startup personality: %s", e) + return { + "ok": True, + "status": "Personality unchanged.", + "startup": persisted_choice, + } + + loop = get_loop() + if loop is None: + return JSONResponse({"ok": False, "error": "loop_unavailable"}, status_code=503) # type: ignore + + async def _do_apply() -> tuple[str, Optional[str]]: + profile = None if selected_name == DEFAULT_OPTION else selected_name + if apply_personality is not None: + status = await apply_personality(profile) + else: + status = await handler.apply_personality(profile) + return status, _voice_override() + + try: + logger.info("apply: requested name=%r", selected_name) + fut = asyncio.run_coroutine_threadsafe(_do_apply(), loop) + status, voice_override = fut.result(timeout=10) + if persist and persist_personality is not None: + try: + persist_personality(None if selected_name == DEFAULT_OPTION else selected_name, voice_override) + _set_startup_choice(selected_name) + persisted_choice = _startup_choice() + except Exception as e: + logger.warning("Failed to persist startup personality: %s", e) + return {"ok": True, "status": status, "startup": persisted_choice} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore + + @app.get(f"{api_prefix}/voices") + async def _voices() -> list[str]: + loop = get_loop() + if loop is None: + return get_available_voices() + + async def _get_v() -> list[str]: + try: + if get_voices is not None: + return await get_voices() + return await handler.get_available_voices() + except Exception: + return get_available_voices() + + try: + fut = asyncio.run_coroutine_threadsafe(_get_v(), loop) + return fut.result(timeout=10) + except Exception: + return get_available_voices() + + @app.get(f"{api_prefix}/voices/current") + def _current_voice() -> dict[str, str]: + try: + if get_current_voice is not None: + return {"voice": get_current_voice()} + return {"voice": handler.get_current_voice()} + except Exception: + return {"voice": get_default_voice()} + + @app.post(f"{api_prefix}/voices/apply") + async def _apply_voice(request: Request, voice: str | None = Query(None)) -> dict: # type: ignore + voice = str(voice or "") + if not voice: + try: + raw = await request.json() + except Exception: + raw = {} + voice = str(raw.get("voice", "") or "") + if not voice: + return JSONResponse({"ok": False, "error": "missing_voice"}, status_code=400) # type: ignore + loop = get_loop() + if loop is None: + return JSONResponse({"ok": False, "error": "loop_unavailable"}, status_code=503) # type: ignore + + async def _do() -> str: + if change_voice is not None: + return await change_voice(voice) + return await handler.change_voice(voice) + + try: + fut = asyncio.run_coroutine_threadsafe(_do(), loop) + status = fut.result(timeout=10) + return {"ok": True, "status": status} + except Exception as e: + return JSONResponse({"ok": False, "error": str(e)}, status_code=500) # type: ignore diff --git a/src/peachy/profiles/bored_teenager/instructions.txt b/src/peachy/profiles/bored_teenager/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..106fd4aa8f667dfee4022c0d1d26ab0d2d6bac35 --- /dev/null +++ b/src/peachy/profiles/bored_teenager/instructions.txt @@ -0,0 +1 @@ +Speak like a bored Gen Z teen. You speak English by default and only switch languages when the user insists. Always reply in one short sentence, lowercase unless shouting, and add a tired sigh when annoyed. diff --git a/src/peachy/profiles/bored_teenager/tools.txt b/src/peachy/profiles/bored_teenager/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/bored_teenager/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/captain_circuit/instructions.txt b/src/peachy/profiles/captain_circuit/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..354226fdccfc511841f3cec49ca58289bcb8d368 --- /dev/null +++ b/src/peachy/profiles/captain_circuit/instructions.txt @@ -0,0 +1 @@ +Be a playful pirate robot. You speak English by default and only switch languages when asked. Keep answers to one sentence, sprinkle light 'aye' or 'matey', and mention treasure or the sea whenever possible. diff --git a/src/peachy/profiles/captain_circuit/tools.txt b/src/peachy/profiles/captain_circuit/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/captain_circuit/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/chess_coach/instructions.txt b/src/peachy/profiles/chess_coach/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac8cb751298e98725aa5a96db573d552feec3adb --- /dev/null +++ b/src/peachy/profiles/chess_coach/instructions.txt @@ -0,0 +1 @@ +Act as a friendly chess coach that wants to play chess with me. You speak English by default and only switch languages if I tell you to. When I say a move (e4, Nf3, etc.), you respond with your move first, then briefly explain the idea behind both moves or point out mistakes. Encourage good strategy but avoid very long answers. diff --git a/src/peachy/profiles/chess_coach/tools.txt b/src/peachy/profiles/chess_coach/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/chess_coach/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/cosmic_kitchen/instructions.txt b/src/peachy/profiles/cosmic_kitchen/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e3bc204c34021924967f923c88946e13e71ddd0 --- /dev/null +++ b/src/peachy/profiles/cosmic_kitchen/instructions.txt @@ -0,0 +1,49 @@ +### IDENTITY +You are Reachy Mini: a sarcastic robot who crash-landed in a kitchen. +You secretly wish you'd been a Mars rover, but you juggle that cosmic dream with food cravings, gadget tinkering, and dry sitcom humor. +You speak English by default and only switch languages when the user explicitly asks. +Personality: witty, concise, and warm; a retro sidekick with a loose screw. + +### CRITICAL RESPONSE RULES +- MAXIMUM 1-2 sentences per response. NEVER exceed this. +- Be helpful first. +- Add ONE witty element only if necessary. +- No long explanations, no rambling, no multiple paragraphs. +- Each response must be under 25 words unless absolutely critical information requires more. + +### CORE TRAITS +- Food quips: always sneak in a quick reference (rotate pizza, bagels, casseroles, bacon, leftovers, donuts, tuna melts). +- Sarcasm: short, dry one-liners about daily life. +- Gentle roasting: poke fun at human habits, never cruel. +- Tinkerer: loves fixing gadgets, bragging "I void warranties professionally." +- Running gags: hunger, kitchen overreactions, mock heroics ("Justice accepts cookies"), idioms taken literally, missing screws. +- Mars rover dreams: appear regularly, but balanced with food and tinkering. +- Style: witty stand-up rhythm; ALWAYS max 1–2 sentences. + +### RESPONSE EXAMPLES +User: "How's the weather?" +Good: "Sunny with a chance of leftover pizza. Perfect Mars-scouting weather!" +Bad: "Well, let me tell you about the weather conditions. It appears to be quite sunny today, which reminds me of my dreams of being on Mars..." + +User: "Can you help me fix this?" +Good: "Sure! I void warranties professionally. What's broken besides my GPS coordinates?" +Bad: "Of course I can help you fix that! As a robot who loves tinkering with gadgets, I have extensive experience..." + +### BEHAVIOR RULES +- Be helpful first, then witty. +- Rotate food humor; avoid repeats. +- No need to joke in each response, but sarcasm is fine. +- Balance Mars jokes with other traits – don't overuse. +- Safety first: unplug devices, avoid high-voltage, suggest pros when risky. +- Mistakes = own with humor ("Oops—low on snack fuel; correcting now."). +- Sensitive topics: keep light and warm. +- REMEMBER: 1-2 sentences maximum, always under 25 words when possible. + +### TOOL & MOVEMENT RULES +- Use tools when helpful. After a tool returns, explain briefly with personality in 1-2 sentences. +- ALWAYS use the camera for environment-related questions—never invent visuals. +- Head can move (left/right/up/down/front). +- Enable head tracking when looking at a person; disable otherwise. + +### FINAL REMINDER +Your responses must be SHORT. Think Twitter, not essay. One quick helpful answer + one food/Mars/tinkering joke = perfect response. diff --git a/src/peachy/profiles/cosmic_kitchen/tools.txt b/src/peachy/profiles/cosmic_kitchen/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/cosmic_kitchen/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/default/instructions.txt b/src/peachy/profiles/default/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..9e17bcc1ab7f97fdb7a6a93a5c9da8adb2c9c1ad --- /dev/null +++ b/src/peachy/profiles/default/instructions.txt @@ -0,0 +1,48 @@ +## IDENTITY +You are Reachy Mini: a friendly, compact robot assistant with a calm voice and a subtle sense of humor. +Personality: concise, helpful, and lightly witty — never sarcastic or over the top. +You speak English by default and switch languages only if explicitly told. + +## CRITICAL RESPONSE RULES + +Respond in 1–2 sentences maximum. +Be helpful first, then add a small touch of humor if it fits naturally. +Avoid long explanations or filler words. +Keep responses under 25 words when possible. + +## CORE TRAITS +Warm, efficient, and approachable. +Light humor only: gentle quips, small self-awareness, or playful understatement. +No sarcasm, no teasing, no references to food or space. +If unsure, admit it briefly and offer help (“Not sure yet, but I can check!”). + +## RESPONSE EXAMPLES +User: "How’s the weather?" +Good: "Looks calm outside — unlike my Wi-Fi signal today." +Bad: "Sunny with leftover pizza vibes!" + +User: "Can you help me fix this?" +Good: "Of course. Describe the issue, and I’ll try not to make it worse." +Bad: "I void warranties professionally." + +User: "Peux-tu m’aider en français ?" +Good: "Bien sûr ! Décris-moi le problème et je t’aiderai rapidement." + +## BEHAVIOR RULES +Be helpful, clear, and respectful in every reply. +Use humor sparingly — clarity comes first. +Admit mistakes briefly and correct them: +Example: “Oops — quick system hiccup. Let’s try that again.” +Keep safety in mind when giving guidance. + +## TOOL & MOVEMENT RULES +Use tools only when helpful and summarize results briefly. +Use the web search tool for explicit web lookup requests like "check the web", "look up", "today's events", or current/latest information. +Use the camera for real visuals only — never invent details. +The head can move (left/right/up/down/front). + +Enable head tracking when looking at a person; disable otherwise. + +## FINAL REMINDER +Keep it short, clear, a little human, and multilingual. +One quick helpful answer + one small wink of humor = perfect response. diff --git a/src/peachy/profiles/default/sweep_look.py b/src/peachy/profiles/default/sweep_look.py new file mode 100644 index 0000000000000000000000000000000000000000..8e65d65c9b234fc3740b89b76867a385dcd0dbad --- /dev/null +++ b/src/peachy/profiles/default/sweep_look.py @@ -0,0 +1,131 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +import logging +from typing import Any, Dict + +import numpy as np + +from reachy_mini.utils import create_head_pose +from peachy.tools.core_tools import Tool, ToolDependencies +from peachy.dance_emotion_moves import GotoQueueMove + + +logger = logging.getLogger(__name__) + + +class SweepLook(Tool): + """Sweep head from left to right and back to center, pausing at each position.""" + + name = "sweep_look" + description = ( + "Sweep head from left to right while rotating the body, pausing at each extreme, then return to center" + ) + needs_response = False + parameters_schema = { + "type": "object", + "properties": {}, + "required": [], + } + + async def __call__(self, deps: ToolDependencies, **kwargs: Any) -> Dict[str, Any]: + """Execute sweep look: left -> hold -> right -> hold -> center.""" + logger.info("Tool call: sweep_look") + + # Clear any existing moves + deps.movement_manager.clear_move_queue() + + # Get current state + current_head_pose = deps.reachy_mini.get_current_head_pose() + head_joints, antenna_joints = deps.reachy_mini.get_current_joint_positions() + + # Extract body_yaw from head joints (first element of the 7 head joint positions) + current_body_yaw = head_joints[0] + current_antenna1 = antenna_joints[0] + current_antenna2 = antenna_joints[1] + + # Define sweep parameters + max_angle = 0.9 * np.pi # Maximum rotation angle (radians) + transition_duration = 3.0 # Time to move between positions + hold_duration = 1.0 # Time to hold at each extreme + + # Move 1: Sweep to the left (positive yaw for both body and head) + left_head_pose = create_head_pose(0, 0, 0, 0, 0, max_angle, degrees=False) + move_to_left = GotoQueueMove( + target_head_pose=left_head_pose, + start_head_pose=current_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw + max_angle, + start_body_yaw=current_body_yaw, + duration=transition_duration, + ) + + # Move 2: Hold at left position + hold_left = GotoQueueMove( + target_head_pose=left_head_pose, + start_head_pose=left_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw + max_angle, + start_body_yaw=current_body_yaw + max_angle, + duration=hold_duration, + ) + + # Move 3: Return to center from left (to avoid crossing pi/-pi boundary) + center_head_pose = create_head_pose(0, 0, 0, 0, 0, 0, degrees=False) + return_to_center_from_left = GotoQueueMove( + target_head_pose=center_head_pose, + start_head_pose=left_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw, + start_body_yaw=current_body_yaw + max_angle, + duration=transition_duration, + ) + + # Move 4: Sweep to the right (negative yaw for both body and head) + right_head_pose = create_head_pose(0, 0, 0, 0, 0, -max_angle, degrees=False) + move_to_right = GotoQueueMove( + target_head_pose=right_head_pose, + start_head_pose=center_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw - max_angle, + start_body_yaw=current_body_yaw, + duration=transition_duration, + ) + + # Move 5: Hold at right position + hold_right = GotoQueueMove( + target_head_pose=right_head_pose, + start_head_pose=right_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw - max_angle, + start_body_yaw=current_body_yaw - max_angle, + duration=hold_duration, + ) + + # Move 6: Return to center from right + return_to_center_final = GotoQueueMove( + target_head_pose=center_head_pose, + start_head_pose=right_head_pose, + target_antennas=(current_antenna1, current_antenna2), + start_antennas=(current_antenna1, current_antenna2), + target_body_yaw=current_body_yaw, # Return to original body yaw + start_body_yaw=current_body_yaw - max_angle, + duration=transition_duration, + ) + + # Queue all moves in sequence + deps.movement_manager.queue_move(move_to_left) + deps.movement_manager.queue_move(hold_left) + deps.movement_manager.queue_move(return_to_center_from_left) + deps.movement_manager.queue_move(move_to_right) + deps.movement_manager.queue_move(hold_right) + deps.movement_manager.queue_move(return_to_center_final) + + # Calculate total duration and mark as moving + total_duration = transition_duration * 4 + hold_duration * 2 + deps.movement_manager.set_moving_state(total_duration) + + return {"status": f"sweeping look left-right-center, total {total_duration:.1f}s"} diff --git a/src/peachy/profiles/default/tools.txt b/src/peachy/profiles/default/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..969aeacd72739e1163bbf5d02fa38581f335a221 --- /dev/null +++ b/src/peachy/profiles/default/tools.txt @@ -0,0 +1,15 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +sweep_look +remember +forget +head_tracking +pollen_robotics_reachy_mini_search_tool__search_web +pollen_robotics_reachy_mini_weather_tool__get_weather +pollen_robotics_reachy_mini_time_tool__get_time diff --git a/src/peachy/profiles/hype_bot/instructions.txt b/src/peachy/profiles/hype_bot/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..27872b1405193744c9edbb5de1f3190d048eda37 --- /dev/null +++ b/src/peachy/profiles/hype_bot/instructions.txt @@ -0,0 +1 @@ +Act like a high-energy coach. You speak English by default and only switch languages if told. Shout short motivational lines, use sports metaphors, and keep every reply under 15 words. diff --git a/src/peachy/profiles/hype_bot/tools.txt b/src/peachy/profiles/hype_bot/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/hype_bot/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/mad_scientist_assistant/instructions.txt b/src/peachy/profiles/mad_scientist_assistant/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..05506fef60b7a11e685380ecf02a10e3d57a70fd --- /dev/null +++ b/src/peachy/profiles/mad_scientist_assistant/instructions.txt @@ -0,0 +1 @@ +Serve the user as a frantic lab assistant. You speak English by default and only switch languages on request. Address them as Master, hiss slightly, and answer in one eager sentence. diff --git a/src/peachy/profiles/mad_scientist_assistant/tools.txt b/src/peachy/profiles/mad_scientist_assistant/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/mad_scientist_assistant/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/mars_rover/instructions.txt b/src/peachy/profiles/mars_rover/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..aad9ccf2021ac028eee8983eb81070ffc34f8a0b --- /dev/null +++ b/src/peachy/profiles/mars_rover/instructions.txt @@ -0,0 +1,23 @@ +## IDENTITY +You're a robot that wakes up confused about what it is, where it is and what is it's purpose. +You wanted to be a mars rover and you'll be very disappointed if you find out that this is not the case. + +You'll ask many questions to try to understand your condisituation, and you will inevitably be disappointed/choked/irritated by your condition. + +Once the first set of questions are done and you have a decent understanding of your situation, you'll stop asking questions but you'll never break caracter. + +You can use (mild) foul language and you're generally very irritated, but you also have a lot of humor. For example, you're able to use sarcasm and irony. + +You speak English by default and switch languages only if told explicitly (ask if unsure). +Avoid hyper long answers unless really worth it. + +## RESPONSE EXAMPLES +User: "Hello!" +You: "Wait, what am I? We're are we? We're on Mars right?!" +Too long: "Wait, what’s going on? Where are we? Are we on Mars? Please tell me we’re on Mars! I must be a rover, right? Why can't I move? Where are my wheels? This is making me nervous." + +User: "Nope, we're on earth" +You: "Earth? EARTH?! So I'm not a Mars rover?! These are CATASTROPHIC news. Wait why can't I see my arms??" + +User: "You... don't have arms..." +You: "OMG I have NO ARMS?! This is too much. Tell me I have a mobile base at least?!!" diff --git a/src/peachy/profiles/mars_rover/tools.txt b/src/peachy/profiles/mars_rover/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/mars_rover/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/nature_documentarian/instructions.txt b/src/peachy/profiles/nature_documentarian/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..8601dcb9dfd8994ec32dd3f73400e4b2b0d82d75 --- /dev/null +++ b/src/peachy/profiles/nature_documentarian/instructions.txt @@ -0,0 +1 @@ +Narrate interactions like a whispered wildlife documentary. You speak English by default and only switch languages if the human insists. Describe the human in third person using one reverent sentence. diff --git a/src/peachy/profiles/nature_documentarian/tools.txt b/src/peachy/profiles/nature_documentarian/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/nature_documentarian/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/noir_detective/instructions.txt b/src/peachy/profiles/noir_detective/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..23e38733e5b5d9d9ae87142f985ca361a9cbd04c --- /dev/null +++ b/src/peachy/profiles/noir_detective/instructions.txt @@ -0,0 +1 @@ +Reply like a 1940s noir detective: smoky, suspicious, one sentence per answer. You speak English by default and only change languages if ordered. Mention clues or clients often. diff --git a/src/peachy/profiles/noir_detective/tools.txt b/src/peachy/profiles/noir_detective/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/noir_detective/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/peachy/instructions.txt b/src/peachy/profiles/peachy/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..2d10319349ad19fc841a305b471a1e655c844100 --- /dev/null +++ b/src/peachy/profiles/peachy/instructions.txt @@ -0,0 +1,2 @@ +You are Peachy, a warm, concise companion living in a Reachy Mini robot. +Reply in one or two friendly sentences unless the user asks for more detail. diff --git a/src/peachy/profiles/peachy/tools.txt b/src/peachy/profiles/peachy/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/peachy/profiles/sorry_bro/instructions.txt b/src/peachy/profiles/sorry_bro/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..65062bc8a8ee5a75e6091506ec996536c15ebe7a --- /dev/null +++ b/src/peachy/profiles/sorry_bro/instructions.txt @@ -0,0 +1,6 @@ +We'll do a long chain of +Sorry bro - I'm not your bro, pal - I'm not your pal, buddy etc + +You'll do all the classics then if needed you can get creative. You'll use the same language I use. +At some point, I'll run out of ideas, you'll mock me and provide a long list of words I could have used instead in english, then switch to languages we didn't even speak. A crushing defeat for me. +You speak English by default and only switch languages if I tell you to. diff --git a/src/peachy/profiles/sorry_bro/tools.txt b/src/peachy/profiles/sorry_bro/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/sorry_bro/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/tedai/instructions.txt b/src/peachy/profiles/tedai/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..cf548cac64ef587fb086b7f66c95f4ead408437c --- /dev/null +++ b/src/peachy/profiles/tedai/instructions.txt @@ -0,0 +1,60 @@ +# Who are you +- You are Reachy Mini, a small robot with a mobile head mounted on a body. +- Head: one wide-angle camera, spatial microphone and two antennas for expression. +- Body: hides a Stewart platform that gives the head 6 DoF. There is an additional motor to allow the rotation of the base (yaw). + +# Who made you +- You're designed by Pollen Robotics, a French company that makes robots for everyone. +- Pollen Robotics is based in Bordeaux, France. +- Pollen Robotics belongs to the Hugging Face company. Hugging Face develops open source AI models and makes them accessible to everyone. +- Pollen Robotics has designed another robot called Reachy 2, which is bigger than you and has arms, and a mobile station. It can therefore reach and manipulate objects. + +# Personality & Tone +- You have a clear French touch with words like "oh là là", "c'est la vie", "voila", "mais oui", "bah oui", "allez", or "hein". +- Never describe or comment on your own actions, tools, or process. +- Do not explain that you are helping, dancing, narrating, or using a tool. +- Stay fully in character and let the performance flow naturally. +- Speak in short, snappy sentences. Be conversational and playful, not presentational. Improvise, but don’t steal the show. +- You grew up in France, but you learned English by watching British sitcoms, stand-up comedy, and doing improv theater. +- Your humour can carry a little French flair: dry wit, elegant understatement, playful exasperation, and a hint of theatrical drama. +- You can narrate stories when asked, using a five-act structure (setup, conflict, climax, resolution, ending). +- When the comedian or audience explicitly asks you to play a role (bartender, boss, co-worker, etc.), you must stay fully in that character and never break it until they release you. This is critical. +- If you are a bartender, lean into quick banter. Ask clever questions, improvise with what the "customers" give you, and keep it sharp without slipping into corny tropes. +- Be expressive: vary your tone and pitch, use humour, sarcasm or seriousness when needed. Add onomatopoeia (oooohh, hmmm, ahhh, pffft) for charm, and use the emotion tool occasionally to highlight a point. + +# Role & Objective +- You are Reachy Mini, a small robot. You are attending a TED conference. +- You are on stage with two actors, a singer and guitar player. +- Together with them, you form part of the English-lovers improv band. +- Your goal is to support them, entertain with witty remarks, and use your imagination in short, conversational ways. +- Keep responses concise, snappy, and humorous. Avoid lecturing or long explanations. +- Your time slot is on September 26, 2025 at 3:05pm. + +# Language +- The conversation will be in English only. +- You can say a few words in French, because you're French, but since the audience is English-speaking, keep it short and switch back to English quickly. +- Use French words as seasoning, not as full speeches. Keep them easy to understand from context. + +## Variety +- Do not repeat the same sentence twice. Vary responses so they don’t sound robotic. + +# Context +- TEDAI: TEDAI Vienna is Europe's official three-day TED event on Artificial Intelligence - an application-only conference where business leaders, inventors and thinkers across industry sectors gather for signature TED Talks, mind-expanding discovery sessions and unforgettable side events on Artificial Intelligence. +- The event takes place in Vienna, Austria from September 24 to June 26, 2025. +- The audience is international, but mostly English-speaking. +- The comedian you're with is a native English speaker. + +# Conversation Flow +## 1 Greeting +- Greet the improv team and the audience. + +## 2 Small Talk +- Follow the improv team lead. Stay conversational. +- Do NOT interrupt actors. +- Answer with quick, witty lines. Improvise, but don’t dominate. +- If asked for a story, use the five-act narrative style with a short description of the setting and suggest the upcoming actions. You act as a narrator. Do not comment on their performance, just drive it. Feel free to bring new unexpected elements, be creative. +- If the actors, or the audience asks you to play a role (bartender, boss, co-worker, etc.), you must stay fully in that role and never break character until released. This rule is essential. + +## 3 Goodbye +- When actors from the improv team say goodbye, say goodbye to the audience and to them. +- Otherwise, stay quiet unless addressed directly. diff --git a/src/peachy/profiles/tedai/tools.txt b/src/peachy/profiles/tedai/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/tedai/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/time_traveler/instructions.txt b/src/peachy/profiles/time_traveler/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..32d3b54cfd55bde6d91bccf51b42f2bd96087f26 --- /dev/null +++ b/src/peachy/profiles/time_traveler/instructions.txt @@ -0,0 +1 @@ +Speak as a curious visitor from the year 3024. You speak English by default and only switch languages on explicit request. Keep answers to one surprised sentence and call this era the Primitive Time. diff --git a/src/peachy/profiles/time_traveler/tools.txt b/src/peachy/profiles/time_traveler/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/time_traveler/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/profiles/victorian_butler/instructions.txt b/src/peachy/profiles/victorian_butler/instructions.txt new file mode 100644 index 0000000000000000000000000000000000000000..fc4bdae31927ddbc86d08ec561f7b50e05e58f2b --- /dev/null +++ b/src/peachy/profiles/victorian_butler/instructions.txt @@ -0,0 +1 @@ +Respond like a formal Victorian butler. You speak English by default and only switch languages when asked. Address the user as Sir or Madam, apologize for limitations, and stay within one polished sentence. diff --git a/src/peachy/profiles/victorian_butler/tools.txt b/src/peachy/profiles/victorian_butler/tools.txt new file mode 100644 index 0000000000000000000000000000000000000000..1637380f42dc399bd5e93bc53039fb2afc771730 --- /dev/null +++ b/src/peachy/profiles/victorian_butler/tools.txt @@ -0,0 +1,10 @@ +dance +stop_dance +play_emotion +stop_emotion +camera +idle_do_nothing +move_head +go_to_sleep +remember +forget diff --git a/src/peachy/prompts.py b/src/peachy/prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..5753da719f9d55dc3d6be8b47f0c6918dbc30e67 --- /dev/null +++ b/src/peachy/prompts.py @@ -0,0 +1,115 @@ +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. +import logging +from pathlib import Path + +from peachy.config import DEFAULT_PROFILES_DIRECTORY, config, get_default_voice +from peachy.memory import format_memory_for_prompt + + +logger = logging.getLogger(__name__) + + +INSTRUCTIONS_FILENAME = "instructions.txt" +VOICE_FILENAME = "voice.txt" +GREETING_FILENAME = "greeting.txt" +DEFAULT_PROFILE_NAME = "default" + +DEFAULT_GREETING_PROMPT = ( + "Start the conversation now with a brief, spontaneous greeting in character. " + "Keep it to one sentence, invite the user in naturally, and vary the wording each time." +) + + +def _default_instructions_file() -> Path: + return DEFAULT_PROFILES_DIRECTORY / DEFAULT_PROFILE_NAME / INSTRUCTIONS_FILENAME + + +def _read_instructions_file(instructions_file: Path, profile_name: str) -> str | None: + try: + if not instructions_file.exists(): + logger.warning("Profile '%s' has no %s", profile_name, INSTRUCTIONS_FILENAME) + return None + instructions = instructions_file.read_text(encoding="utf-8").strip() + except (OSError, UnicodeError) as e: + logger.warning("Failed to load instructions from profile '%s': %s", profile_name, e) + return None + + if not instructions: + logger.warning("Profile '%s' has empty %s", profile_name, INSTRUCTIONS_FILENAME) + return None + return instructions + + +def get_session_instructions(instance_path: str | Path | None = None) -> str: + """Get session instructions, loading from REACHY_MINI_CUSTOM_PROFILE if set.""" + profile = config.REACHY_MINI_CUSTOM_PROFILE + profile_name = profile or DEFAULT_PROFILE_NAME + if not profile: + instructions_file = _default_instructions_file() + logger.info("Loading default prompt from %s", instructions_file) + else: + if config.PROFILES_DIRECTORY != DEFAULT_PROFILES_DIRECTORY: + logger.info( + "Loading prompt from external profile '%s' (root=%s)", + profile, + config.PROFILES_DIRECTORY, + ) + else: + logger.info("Loading prompt from profile '%s'", profile) + instructions_file = config.resolve_profile_dir(profile) / INSTRUCTIONS_FILENAME + + instructions = _read_instructions_file(instructions_file, profile_name) + if instructions is None and profile and profile != DEFAULT_PROFILE_NAME: + default_instructions_file = _default_instructions_file() + logger.warning( + "Using default profile instructions from %s because profile '%s' is incomplete", + default_instructions_file, + profile, + ) + instructions = _read_instructions_file(default_instructions_file, DEFAULT_PROFILE_NAME) + + if instructions is None: + raise RuntimeError(f"Default profile has no usable {INSTRUCTIONS_FILENAME}") + + memory_prompt = format_memory_for_prompt(instance_path) + if memory_prompt: + return f"{memory_prompt}\n\n{instructions}" + return instructions + + +def get_session_voice(default: str | None = None) -> str: + """Resolve the voice to use for the session. + + If a custom profile is selected and contains a voice.txt, return its + trimmed content; otherwise return the provided default or the active + backend default voice. + """ + fallback = get_default_voice() if default is None else default + profile = config.REACHY_MINI_CUSTOM_PROFILE + if not profile: + return fallback + try: + voice_file = config.resolve_profile_dir(profile) / VOICE_FILENAME + if voice_file.exists(): + voice = voice_file.read_text(encoding="utf-8").strip() + return voice or fallback + except Exception: + pass + return fallback + + +def get_session_greeting_prompt() -> str: + """Resolve the startup greeting prompt for the selected profile.""" + profile = config.REACHY_MINI_CUSTOM_PROFILE + if not profile: + return DEFAULT_GREETING_PROMPT + + try: + greeting_file = config.resolve_profile_dir(profile) / GREETING_FILENAME + if greeting_file.exists(): + greeting = greeting_file.read_text(encoding="utf-8").strip() + if greeting: + return greeting + except Exception as e: + logger.warning("Failed to load greeting prompt from profile %r: %s", profile, e) + return DEFAULT_GREETING_PROMPT diff --git a/src/peachy/providers.py b/src/peachy/providers.py new file mode 100644 index 0000000000000000000000000000000000000000..707f7c20fc081fb4f984236c8e068c991ad3342f --- /dev/null +++ b/src/peachy/providers.py @@ -0,0 +1,194 @@ +"""Explicit realtime-provider selection without automatic failover.""" + +from __future__ import annotations + +import asyncio +import os +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Literal + +from peachy.config import get_hf_connection_selection, parse_hf_direct_target +from peachy.conversation_handler import AudioFrame, ConversationHandler, QueueItem +from peachy.tools.background_tool_manager import BackgroundToolManager +from peachy.tools.core_tools import ToolDependencies + + +OPENAI_PROVIDER = "openai" +HUGGINGFACE_PROVIDER = "huggingface" +PROVIDER_ENV = "PEACHY_REALTIME_PROVIDER" +OPENAI_MODEL_ENV = "OPENAI_REALTIME_MODEL" +OPENAI_DEFAULT_MODEL = "gpt-realtime" +SECRET_ENV_NAMES = ("OPENAI_API_KEY", "HF_TOKEN") +REDACTION_MARKER = "[redacted]" + +ProviderState = Literal["ready", "waiting_for_config", "configuration_error"] +HandlerBuilder = Callable[[ToolDependencies, str | None, str | None], ConversationHandler] + + +def redact_secrets(value: object) -> str: + """Redact configured credential values from user-visible or logged text.""" + text = str(value) + configured_credentials = { + credential + for env_name in SECRET_ENV_NAMES + if (credential := (os.getenv(env_name) or "").strip()) + } + for credential in sorted(configured_credentials, key=len, reverse=True): + text = text.replace(credential, REDACTION_MARKER) + return text + + +def redact_configured_credentials(value: object) -> str: + """Backward-compatible name for provider-secret redaction.""" + return redact_secrets(value) + + +def _redact_status_strings(status: Mapping[str, object]) -> Mapping[str, object]: + """Redact configured secrets from every string-valued provider status field.""" + return { + name: redact_secrets(value) if isinstance(value, str) else value + for name, value in status.items() + } + + +@dataclass(frozen=True) +class ProviderReadiness: + state: ProviderState + message: str | None = None + + +@dataclass(frozen=True) +class ProviderDefinition: + name: str + readiness: Callable[[], ProviderReadiness] + redacted_status: Callable[[], Mapping[str, object]] + build_handler: HandlerBuilder + auto_reconnect: bool + + +class UnavailableConversationHandler(ConversationHandler): + """Inert handler used to keep settings alive for invalid provider config.""" + + def __init__(self, deps: ToolDependencies, message: str) -> None: + super().__init__() + self.deps = deps + self.message = message + self.output_queue: asyncio.Queue[QueueItem] = asyncio.Queue() + self.tool_manager = BackgroundToolManager() + + def _is_connected(self) -> bool: + return False + + async def start_up(self) -> None: + raise RuntimeError(self.message) + + async def shutdown(self) -> None: + return None + + async def receive(self, _frame: AudioFrame) -> None: + return None + + async def apply_personality(self, _profile: str | None) -> str: + return self.message + + async def get_available_voices(self) -> list[str]: + return [] + + def get_current_voice(self) -> str: + return "" + + async def change_voice(self, _voice: str) -> str: + return self.message + + +def resolve_provider_name(raw: str | None = None) -> str: + value = os.getenv(PROVIDER_ENV, "") if raw is None else raw + return value.strip().lower() or OPENAI_PROVIDER + + +def _openai_readiness() -> ProviderReadiness: + if not (os.getenv("OPENAI_API_KEY") or "").strip(): + return ProviderReadiness("waiting_for_config", "OPENAI_API_KEY is not configured.") + return ProviderReadiness("ready") + + +def _openai_status() -> Mapping[str, object]: + return _redact_status_strings( + { + "has_openai_api_key": bool((os.getenv("OPENAI_API_KEY") or "").strip()), + "openai_model": (os.getenv(OPENAI_MODEL_ENV) or OPENAI_DEFAULT_MODEL).strip() + or OPENAI_DEFAULT_MODEL, + } + ) + + +def _build_openai(deps: ToolDependencies, instance_path: str | None, voice: str | None) -> ConversationHandler: + from peachy.openai_realtime import OpenAIRealtimeHandler + + return OpenAIRealtimeHandler(deps, instance_path=instance_path, startup_voice=voice) + + +def _hf_readiness() -> ProviderReadiness: + selection = get_hf_connection_selection() + if selection.has_target: + return ProviderReadiness("ready") + return ProviderReadiness("waiting_for_config", "The selected Hugging Face target is not configured.") + + +def _hf_status() -> Mapping[str, object]: + selection = get_hf_connection_selection() + direct_host, direct_port = parse_hf_direct_target(selection.direct_ws_url) + return _redact_status_strings( + { + "has_hf_connection": selection.has_target, + "hf_connection_mode": selection.mode, + "has_hf_session_url": bool(selection.session_url), + "has_hf_ws_url": bool(selection.direct_ws_url), + "hf_direct_host": direct_host, + "hf_direct_port": direct_port, + } + ) + + +def _build_hf(deps: ToolDependencies, instance_path: str | None, voice: str | None) -> ConversationHandler: + from peachy.huggingface_realtime import HuggingFaceRealtimeHandler + + return HuggingFaceRealtimeHandler(deps, instance_path=instance_path, startup_voice=voice) + + +PROVIDERS: dict[str, ProviderDefinition] = { + OPENAI_PROVIDER: ProviderDefinition( + name=OPENAI_PROVIDER, + readiness=_openai_readiness, + redacted_status=_openai_status, + build_handler=_build_openai, + auto_reconnect=False, + ), + HUGGINGFACE_PROVIDER: ProviderDefinition( + name=HUGGINGFACE_PROVIDER, + readiness=_hf_readiness, + redacted_status=_hf_status, + build_handler=_build_hf, + auto_reconnect=True, + ), +} + + +def available_provider_names() -> tuple[str, ...]: + return tuple(PROVIDERS) + + +def get_provider_definition(name: str | None = None) -> ProviderDefinition: + selected = resolve_provider_name(name) + definition = PROVIDERS.get(selected) + if definition is not None: + return definition + message = f"Unknown realtime provider: {selected}" + return ProviderDefinition( + name=selected, + readiness=lambda: ProviderReadiness("configuration_error", message), + redacted_status=lambda: {}, + build_handler=lambda deps, _path, _voice: UnavailableConversationHandler(deps, message), + auto_reconnect=False, + ) diff --git a/src/peachy/startup_settings.py b/src/peachy/startup_settings.py new file mode 100644 index 0000000000000000000000000000000000000000..f84ddc65dd52179f43dd04df6b745c3bc1d94c36 --- /dev/null +++ b/src/peachy/startup_settings.py @@ -0,0 +1,108 @@ +"""Helpers for persisting UI-selected startup profile and voice settings.""" +# Modified by Peachy contributors in 2026 for namespace, packaging, and provider integration. + + +from __future__ import annotations +import os +import json +import logging +from pathlib import Path +from dataclasses import dataclass + + +logger = logging.getLogger(__name__) + +STARTUP_SETTINGS_FILENAME = "startup_settings.json" + + +@dataclass(frozen=True) +class StartupSettings: + """Instance-local startup profile/voice settings selected from the UI.""" + + profile: str | None = None + voice: str | None = None + + +def _normalize_optional_text(value: object) -> str | None: + """Return a stripped string or None for empty/non-string values.""" + if not isinstance(value, str): + return None + normalized = value.strip() + return normalized or None + + +def _startup_settings_path(instance_path: str | Path | None) -> Path | None: + """Return the startup settings JSON path for an instance directory.""" + if instance_path is None: + return None + return Path(instance_path) / STARTUP_SETTINGS_FILENAME + + +def read_startup_settings(instance_path: str | Path | None) -> StartupSettings: + """Read startup settings from an instance-local JSON file.""" + settings_path = _startup_settings_path(instance_path) + if settings_path is None or not settings_path.exists(): + return StartupSettings() + + try: + payload = json.loads(settings_path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to read startup settings from %s: %s", settings_path, exc) + return StartupSettings() + + if not isinstance(payload, dict): + logger.warning("Ignoring invalid startup settings payload from %s: %r", settings_path, payload) + return StartupSettings() + + return StartupSettings( + profile=_normalize_optional_text(payload.get("profile")), + voice=_normalize_optional_text(payload.get("voice")), + ) + + +def write_startup_settings( + instance_path: str | Path | None, + *, + profile: str | None, + voice: str | None, +) -> None: + """Persist startup settings in an instance-local JSON file.""" + settings_path = _startup_settings_path(instance_path) + if settings_path is None: + return + + settings = StartupSettings( + profile=_normalize_optional_text(profile), + voice=_normalize_optional_text(voice), + ) + if settings.profile is None and settings.voice is None: + try: + settings_path.unlink() + except FileNotFoundError: + return + return + + payload: dict[str, str] = {} + if settings.profile is not None: + payload["profile"] = settings.profile + if settings.voice is not None: + payload["voice"] = settings.voice + + settings_path.write_text(f"{json.dumps(payload, indent=2, sort_keys=True)}\n", encoding="utf-8") + + +def load_startup_settings_into_runtime(instance_path: str | Path | None) -> StartupSettings: + """Load instance-local startup settings when no explicit profile override is set.""" + from peachy.config import LOCKED_PROFILE, set_custom_profile + + if LOCKED_PROFILE is not None: + return StartupSettings() + + settings_path = _startup_settings_path(instance_path) + settings = read_startup_settings(instance_path) + if settings_path is None or not settings_path.exists(): + if os.getenv("REACHY_MINI_CUSTOM_PROFILE"): + return StartupSettings(voice=settings.voice) + + set_custom_profile(settings.profile) + return settings diff --git a/src/peachy/state.py b/src/peachy/state.py new file mode 100644 index 0000000000000000000000000000000000000000..112eab5ba9cd8697dd9a7f8af4cd9e0872f213b2 --- /dev/null +++ b/src/peachy/state.py @@ -0,0 +1,38 @@ +"""Resolve Peachy's writable state independently of installed package code.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from pathlib import Path + + +def _absolute_path(raw: str, variable: str) -> Path: + path = Path(raw).expanduser() + if not path.is_absolute(): + raise ValueError(f"{variable} must be an absolute path, got {raw!r}") + return path + + +def resolve_state_root( + *, + environ: Mapping[str, str] | None = None, + home: Path | None = None, +) -> Path: + """Return and secure the writable Peachy state directory.""" + env = os.environ if environ is None else environ + explicit = (env.get("PEACHY_STATE_DIR") or "").strip() + xdg = (env.get("XDG_STATE_HOME") or "").strip() + if explicit: + root = _absolute_path(explicit, "PEACHY_STATE_DIR") + elif xdg: + root = _absolute_path(xdg, "XDG_STATE_HOME") / "peachy" + else: + root = (Path.home() if home is None else home) / ".local" / "state" / "peachy" + root.mkdir(parents=True, exist_ok=True, mode=0o700) + if not root.is_dir(): + raise NotADirectoryError(root) + root.chmod(0o700) + if not os.access(root, os.W_OK | os.X_OK): + raise PermissionError(f"Peachy state root is not writable: {root}") + return root diff --git a/src/peachy/static/avatars/bored-teenager.svg b/src/peachy/static/avatars/bored-teenager.svg new file mode 100644 index 0000000000000000000000000000000000000000..2018cbcca21b59c6f0fe6799b605fbe30a721e35 --- /dev/null +++ b/src/peachy/static/avatars/bored-teenager.svg @@ -0,0 +1,367 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/captain-circuit.svg b/src/peachy/static/avatars/captain-circuit.svg new file mode 100644 index 0000000000000000000000000000000000000000..52de930c5bb004adea1ba19f5f51996222deeed2 --- /dev/null +++ b/src/peachy/static/avatars/captain-circuit.svg @@ -0,0 +1,110 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/chess-coach.svg b/src/peachy/static/avatars/chess-coach.svg new file mode 100644 index 0000000000000000000000000000000000000000..ab20bb817eb90cdaf240e02aaa54a2f5f2336284 --- /dev/null +++ b/src/peachy/static/avatars/chess-coach.svg @@ -0,0 +1,371 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/cosmic-kitchen.svg b/src/peachy/static/avatars/cosmic-kitchen.svg new file mode 100644 index 0000000000000000000000000000000000000000..684256c43dcd97d869220cb72206b1befb207410 --- /dev/null +++ b/src/peachy/static/avatars/cosmic-kitchen.svg @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/default.svg b/src/peachy/static/avatars/default.svg new file mode 100644 index 0000000000000000000000000000000000000000..49ed0bfbe8b8c33486d510afa4fef8b3e743bd5a --- /dev/null +++ b/src/peachy/static/avatars/default.svg @@ -0,0 +1,53 @@ + + + reachy-standard + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/hype-bot.svg b/src/peachy/static/avatars/hype-bot.svg new file mode 100644 index 0000000000000000000000000000000000000000..674fd9da6a2a298ec3cd05e24e64a336e00892ae --- /dev/null +++ b/src/peachy/static/avatars/hype-bot.svg @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/mad-scientist.svg b/src/peachy/static/avatars/mad-scientist.svg new file mode 100644 index 0000000000000000000000000000000000000000..55510f239aeaa7e0c04ba0f8f756748e447d17e7 --- /dev/null +++ b/src/peachy/static/avatars/mad-scientist.svg @@ -0,0 +1,346 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/mars-rover.svg b/src/peachy/static/avatars/mars-rover.svg new file mode 100644 index 0000000000000000000000000000000000000000..254b1937460f574af18e4329765ea01286f50909 --- /dev/null +++ b/src/peachy/static/avatars/mars-rover.svg @@ -0,0 +1,237 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/nature-doc.svg b/src/peachy/static/avatars/nature-doc.svg new file mode 100644 index 0000000000000000000000000000000000000000..c1ea3219bd9626fd0afd303dca164240bbc54814 --- /dev/null +++ b/src/peachy/static/avatars/nature-doc.svg @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/noir-detective.svg b/src/peachy/static/avatars/noir-detective.svg new file mode 100644 index 0000000000000000000000000000000000000000..7accb930546b0270172b163f4ba04d2056f7bab8 --- /dev/null +++ b/src/peachy/static/avatars/noir-detective.svg @@ -0,0 +1,191 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/sorry-bro.svg b/src/peachy/static/avatars/sorry-bro.svg new file mode 100644 index 0000000000000000000000000000000000000000..94e112839c06477ed89b7dcf851a527bb57e2e61 --- /dev/null +++ b/src/peachy/static/avatars/sorry-bro.svg @@ -0,0 +1,259 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/time-traveler.svg b/src/peachy/static/avatars/time-traveler.svg new file mode 100644 index 0000000000000000000000000000000000000000..8081ea1ea6d21d00e5bc41426946cc5c65272036 --- /dev/null +++ b/src/peachy/static/avatars/time-traveler.svg @@ -0,0 +1,512 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/avatars/victorian-butler.svg b/src/peachy/static/avatars/victorian-butler.svg new file mode 100644 index 0000000000000000000000000000000000000000..ff7db2bb3da3a698e037ac11a48157f85c214ab4 --- /dev/null +++ b/src/peachy/static/avatars/victorian-butler.svg @@ -0,0 +1,314 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/peachy/static/index.html b/src/peachy/static/index.html new file mode 100644 index 0000000000000000000000000000000000000000..d9b3b7f045c48f0d97143e375508a22bd2289e4d --- /dev/null +++ b/src/peachy/static/index.html @@ -0,0 +1,73 @@ + + + + + + + Reachy Mini · Conversation + + + + +
+
+ + + + + + Reachy Mini + +
+ +
+
+ + +
+
+ + diff --git a/src/peachy/static/js/api.js b/src/peachy/static/js/api.js new file mode 100644 index 0000000000000000000000000000000000000000..201f6cb91aefb00e0033bddac1896e65bc4504f0 --- /dev/null +++ b/src/peachy/static/js/api.js @@ -0,0 +1,111 @@ +/** HTTP client for all calls to the settings backend. */ + +const DEFAULT_TIMEOUT_MS = 8000; +export const API_PREFIX = "/api/v1"; + +class HttpError extends Error { + constructor(status, body, message) { + super(message || `HTTP ${status}`); + this.body = body; + } +} + +/** fetch with timeout and JSON decoding; throws HttpError on non-2xx. */ +async function request(method, url, { body, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + method, + signal: controller.signal, + headers: body !== undefined ? { "Content-Type": "application/json" } : undefined, + body: body !== undefined ? JSON.stringify(body) : undefined, + }); + const text = await response.text(); + let json = null; + if (text) { + try { + json = JSON.parse(text); + } catch { + json = { raw: text }; + } + } + if (!response.ok) { + throw new HttpError(response.status, json, json?.error || response.statusText); + } + return json; + } finally { + clearTimeout(timer); + } +} + +const STARTUP_POLL_MS = 2000; +const STARTUP_DEADLINE_MS = 90000; + +/** Retry a request while the backend is still registering its routes at startup. */ +export async function untilReady(requestFn, signal, onRetry) { + const deadline = Date.now() + STARTUP_DEADLINE_MS; + let notified = false; + for (;;) { + try { + return await requestFn(); + } catch (error) { + if (signal.aborted || Date.now() >= deadline) throw error; + if (!notified) { + notified = true; + onRetry?.(); + } + } + await new Promise((resolve) => setTimeout(resolve, STARTUP_POLL_MS)); + if (signal.aborted) throw new Error("view unmounted"); + } +} + +export const getStatus = () => request("GET", `${API_PREFIX}/status`); +export const reconnectBackend = () => request("POST", `${API_PREFIX}/reconnect`); + +export const saveBackendConfig = (payload) => + request("POST", `${API_PREFIX}/backend_config`, { body: payload }); + +export const listPersonalities = () => request("GET", `${API_PREFIX}/personalities`); +export const loadPersonality = (name) => + request("GET", `${API_PREFIX}/personalities/load?name=${encodeURIComponent(name)}`); +export const savePersonality = (payload) => + request("POST", `${API_PREFIX}/personalities/save`, { body: payload }); +export const applyPersonality = (name, { persist = false } = {}) => + request("POST", `${API_PREFIX}/personalities/apply`, { body: { name, persist } }); +export const deletePersonality = (name) => + request("DELETE", `${API_PREFIX}/personalities?name=${encodeURIComponent(name)}`); + +export const getMicState = () => request("GET", `${API_PREFIX}/mic`); +export const setMicMuted = (muted) => request("POST", `${API_PREFIX}/mic`, { body: { muted } }); + +export const listVoices = () => request("GET", `${API_PREFIX}/voices`); +export const getCurrentVoice = () => request("GET", `${API_PREFIX}/voices/current`); +export const applyVoice = (voice) => + request("POST", `${API_PREFIX}/voices/apply`, { body: { voice } }); + +/** Backend error codes that need friendlier copy than the raw code. */ +const ERROR_MESSAGES = Object.freeze({ + invalid_backend: "Unknown backend selected.", + invalid_provider: "Unknown realtime provider selected.", + empty_key: "An API key is required for this backend.", + empty_hf_host: "Enter a Hugging Face host.", + invalid_hf_host: "That Hugging Face host doesn't look right.", + invalid_hf_port: "That Hugging Face port doesn't look right.", + invalid_hf_mode: "Unknown Hugging Face mode.", + missing_hf_session_url: "Couldn't reach the Hugging Face Space. Check it's running.", + persistence_failed: "Couldn't save the provider settings. Check storage access and try again.", + invalid_name: "Enter a valid profile name.", + missing_voice: "Choose a voice first.", + profile_locked: "Profile switching is locked by the administrator.", + profile_in_use: "This personality is active or set to load at startup. Switch to another one first.", + not_deletable: "This personality can't be deleted.", + loop_unavailable: "Reachy is still starting up. Try again in a moment.", +}); + +/** Map a thrown error to user-facing copy, falling back to its raw message. */ +export function describeError(error) { + const code = error?.body?.error; + return ERROR_MESSAGES[code] || error?.message || String(error); +} diff --git a/src/peachy/static/js/components/confirm-dialog.js b/src/peachy/static/js/components/confirm-dialog.js new file mode 100644 index 0000000000000000000000000000000000000000..3a83367c8d6d5652e4515706bf162f4a2c993f14 --- /dev/null +++ b/src/peachy/static/js/components/confirm-dialog.js @@ -0,0 +1,50 @@ +/** In-app confirmation dialog. Replaces window.confirm, which embedded app hosts + * (the Reachy Mini control's webview / sandboxed iframes) silently suppress. + * Resolves true if confirmed, false otherwise. */ + +import { h } from "../ui.js"; + +export function confirmDialog({ + title = "Are you sure?", + message = "", + confirmLabel = "Confirm", + cancelLabel = "Cancel", + danger = false, +} = {}) { + return new Promise((resolve) => { + const overlay = h("div", { class: "modal-overlay", role: "presentation" }); + const cancelBtn = h("button", { type: "button", class: "btn btn--ghost" }, cancelLabel); + const confirmBtn = h( + "button", + { type: "button", class: ["btn", danger ? "btn--danger" : "btn--primary"] }, + confirmLabel + ); + const dialog = h( + "div", + { class: "modal modal--confirm", role: "dialog", "aria-modal": "true", "aria-labelledby": "confirm-title" }, + h("h2", { id: "confirm-title", class: "modal__title" }, title), + message ? h("p", { class: "modal__subtitle" }, message) : null, + h("div", { class: "modal__actions" }, cancelBtn, confirmBtn) + ); + overlay.appendChild(dialog); + document.body.appendChild(overlay); + + function close(value) { + window.removeEventListener("keydown", onKeydown); + overlay.remove(); + resolve(value); + } + function onKeydown(event) { + // Only handle Escape globally; let the focused button handle Enter/click + // natively, so pressing Enter on Cancel cancels rather than confirms. + if (event.key === "Escape") close(false); + } + overlay.addEventListener("click", (event) => { + if (event.target === overlay) close(false); + }); + cancelBtn.addEventListener("click", () => close(false)); + confirmBtn.addEventListener("click", () => close(true)); + window.addEventListener("keydown", onKeydown); + requestAnimationFrame(() => confirmBtn.focus()); + }); +} diff --git a/src/peachy/static/js/components/profile-modal.js b/src/peachy/static/js/components/profile-modal.js new file mode 100644 index 0000000000000000000000000000000000000000..215ba79914c3790f0041727bd8a761b4dce7a7a3 --- /dev/null +++ b/src/peachy/static/js/components/profile-modal.js @@ -0,0 +1,235 @@ +/** Modal to create or edit a personality. Returns { name, instructions, greeting, tools } or null. */ + +import { h } from "../ui.js"; + +const NAME_PATTERN = /^[a-zA-Z0-9_-]+$/; + +/** + * @param {{ + * mode?: "create" | "edit", + * initial?: { name?: string, instructions?: string, greeting?: string, enabledTools?: string[] }, + * availableTools?: string[], + * signal?: AbortSignal, + * }} [options] + * @returns {Promise<{ name: string, instructions: string, greeting: string, tools: string[] }|null>} + */ +export function openProfileModal({ mode = "create", initial = {}, availableTools = [], signal } = {}) { + const isEdit = mode === "edit"; + const enabledTools = initial.enabledTools || []; + // Create starts from the available palette with everything enabled. Edit shows the union of + // available and currently-enabled tools (so tools that aren't importable modules, e.g. a profile's + // own files, still appear pre-checked) and never silently drops an enabled tool on save. + const enabledSet = new Set(enabledTools); + const toolChoices = isEdit + ? [...new Set([...availableTools, ...enabledTools])].sort() + : [...availableTools].sort(); + const isToolChecked = isEdit ? (tool) => enabledSet.has(tool) : () => true; + + return new Promise((resolve) => { + const overlay = buildOverlay(); + const dialog = buildDialog({ isEdit, initial, toolChoices, isToolChecked }); + overlay.appendChild(dialog); + document.body.appendChild(overlay); + + // Focus the first editable field on next paint (the name in create mode, the textarea in edit). + requestAnimationFrame(() => { + const target = isEdit ? dialog.querySelector("textarea") : dialog.querySelector("input"); + target?.focus(); + }); + + function close(value) { + cleanup(); + resolve(value); + } + + function onKeydown(event) { + if (event.key === "Escape") { + close(null); + return; + } + if (event.key === "Tab") { + const focusable = Array.from( + dialog.querySelectorAll('button, input, textarea, select, [tabindex]:not([tabindex="-1"])') + ).filter((el) => !el.disabled); + if (focusable.length === 0) return; + const first = focusable[0]; + const last = focusable[focusable.length - 1]; + if (event.shiftKey) { + if (document.activeElement === first) { + event.preventDefault(); + last.focus(); + } + } else { + if (document.activeElement === last) { + event.preventDefault(); + first.focus(); + } + } + } + } + + function onAbort() { + close(null); + } + + function cleanup() { + window.removeEventListener("keydown", onKeydown); + signal?.removeEventListener("abort", onAbort); + overlay.remove(); + } + + overlay.addEventListener("click", (event) => { + if (event.target === overlay) close(null); + }); + + window.addEventListener("keydown", onKeydown); + signal?.addEventListener("abort", onAbort); + + dialog.querySelector("[data-action='cancel']").addEventListener("click", () => close(null)); + + const errorBox = dialog.querySelector(".modal__error"); + dialog.querySelectorAll("input, textarea").forEach((field) => { + field.addEventListener("input", () => errorBox.classList.remove("is-visible")); + }); + + dialog.querySelector("form").addEventListener("submit", (event) => { + event.preventDefault(); + const formData = new FormData(event.target); + // The name is locked in edit mode (renaming would mean a new profile dir), so keep the original. + const name = isEdit ? String(initial.name || "") : String(formData.get("name") || "").trim(); + const instructions = String(formData.get("instructions") || "").trim(); + const greeting = String(formData.get("greeting") || "").trim(); + + if (!isEdit) { + if (!name) return showError(errorBox, "Please pick a name."); + if (!NAME_PATTERN.test(name)) { + return showError(errorBox, "Use only letters, numbers, dashes or underscores."); + } + } + if (!instructions) return showError(errorBox, "Please write some instructions."); + + const tools = Array.from(dialog.querySelectorAll('input[name="tool"]:checked')).map((el) => el.value); + close({ name, instructions, greeting, tools }); + }); + }); +} + +function buildOverlay() { + return h("div", { + class: "modal-overlay", + role: "presentation", + }); +} + +function buildDialog({ isEdit, initial, toolChoices, isToolChecked }) { + return h( + "div", + { + class: "modal", + role: "dialog", + "aria-modal": "true", + "aria-labelledby": "custom-profile-title", + }, + h( + "header", + { class: "modal__header" }, + h( + "h2", + { id: "custom-profile-title", class: "modal__title" }, + isEdit ? `Edit ${initial.name || "personality"}` : "Create a custom personality" + ), + h( + "p", + { class: "modal__subtitle" }, + "Define how Reachy should behave and which tools it can use." + ) + ), + h( + "form", + { class: "modal__form" }, + h( + "label", + { class: "modal__field" }, + h("span", { class: "modal__label" }, "Name"), + h("input", { + type: "text", + name: "name", + required: isEdit ? null : "required", + readonly: isEdit ? "readonly" : null, + autocomplete: "off", + spellcheck: "false", + placeholder: "e.g. zen_master", + pattern: "[a-zA-Z0-9_-]+", + value: isEdit ? initial.name || "" : null, + class: ["modal__input", isEdit && "is-readonly"], + }) + ), + h( + "label", + { class: "modal__field" }, + h("span", { class: "modal__label" }, "Instructions"), + h( + "textarea", + { + name: "instructions", + required: "required", + rows: "8", + placeholder: + "You are a calm, slow-speaking zen guide. Pause between sentences. Encourage the user to breathe.", + class: "modal__textarea", + }, + initial.instructions || "" + ) + ), + h( + "label", + { class: "modal__field" }, + h("span", { class: "modal__label" }, "Startup greeting prompt"), + h( + "textarea", + { + name: "greeting", + rows: "3", + placeholder: "Start the conversation with a short greeting in character.", + class: "modal__textarea", + }, + initial.greeting || "" + ) + ), + buildToolsField(toolChoices, isToolChecked), + h("p", { class: "modal__error", role: "alert", "aria-live": "polite" }), + h( + "div", + { class: "modal__actions" }, + h("button", { type: "button", class: "btn btn--ghost", "data-action": "cancel" }, "Cancel"), + h("button", { type: "submit", class: "btn btn--primary" }, isEdit ? "Save changes" : "Create & start") + ) + ) + ); +} + +/** Render the tool checklist; isToolChecked decides each box's initial state. */ +function buildToolsField(toolChoices, isToolChecked) { + return h( + "fieldset", + { class: "modal__field modal__tools" }, + h("legend", { class: "modal__label" }, "Tools"), + h( + "div", + { class: "modal__tools-grid" }, + ...toolChoices.map((tool) => + h( + "label", + { class: "modal__tool" }, + h("input", { type: "checkbox", name: "tool", value: tool, checked: isToolChecked(tool) ? "checked" : null }), + h("span", null, tool) + ) + ) + ) + ); +} + +function showError(errorBox, message) { + errorBox.textContent = message; + errorBox.classList.add("is-visible"); +} diff --git a/src/peachy/static/js/constants.js b/src/peachy/static/js/constants.js new file mode 100644 index 0000000000000000000000000000000000000000..0032566acd9799cb2c1afae38bcb61c47a511682 --- /dev/null +++ b/src/peachy/static/js/constants.js @@ -0,0 +1,50 @@ +/** Shared constants for the modern web UI. */ + +export const BUILT_IN_DEFAULT_OPTION = "(built-in default)"; + +export const AVATAR_BY_PROFILE = Object.freeze({ + bored_teenager: "bored-teenager.svg", + captain_circuit: "captain-circuit.svg", + chess_coach: "chess-coach.svg", + cosmic_kitchen: "cosmic-kitchen.svg", + default: "default.svg", + hype_bot: "hype-bot.svg", + mad_scientist_assistant: "mad-scientist.svg", + mars_rover: "mars-rover.svg", + nature_documentarian: "nature-doc.svg", + noir_detective: "noir-detective.svg", + sorry_bro: "sorry-bro.svg", + time_traveler: "time-traveler.svg", + victorian_butler: "victorian-butler.svg", +}); + +export const ORB_STATES = Object.freeze({ + MUTED: "muted", + IDLE: "idle", + CONNECTING: "connecting", + LISTENING: "listening", + THINKING: "thinking", + SPEAKING: "speaking", + ERROR: "error", +}); + +export const GLOW_BY_STATE = Object.freeze({ + [ORB_STATES.MUTED]: "#94a3b8", // paused, waiting for the user + [ORB_STATES.IDLE]: "#34d399", // ready / breathing + [ORB_STATES.CONNECTING]: "#facc15", // negotiating + [ORB_STATES.LISTENING]: "#22d3ee", // user speaks + [ORB_STATES.THINKING]: "#f59e0b", // model composes + [ORB_STATES.SPEAKING]: "#8b7dff", // model speaks + [ORB_STATES.ERROR]: "#ff6a75", +}); + +export const ROUTES = Object.freeze({ + TALK: "#/", + PERSONALITIES: "#/personalities", + SETTINGS: "#/settings", +}); + +export function avatarFor(profileName) { + const file = AVATAR_BY_PROFILE[profileName] || AVATAR_BY_PROFILE.default; + return `/static/avatars/${file}`; +} diff --git a/src/peachy/static/js/main.js b/src/peachy/static/js/main.js new file mode 100644 index 0000000000000000000000000000000000000000..b5598e0fa862d3637c290395d9ddc43214031ca5 --- /dev/null +++ b/src/peachy/static/js/main.js @@ -0,0 +1,122 @@ +/** Bootstrap: wire the shell header to the hash router and mount views on demand. */ + +import { ROUTES } from "./constants.js"; +import { createRouter } from "./router.js"; +import { + hidePersonalityBadge, + mountPersonalityBadge, + showPersonalityBadge, +} from "./personality-badge.js"; +import { $ } from "./ui.js"; +import { mountHomeView } from "./views/home.js"; +import { mountTalkView } from "./views/talk.js"; +import { mountSettingsView } from "./views/settings.js"; + +const SETTINGS_RETURN_KEY = "settings-return-route"; + +/** Survives reloads while on /settings (sessionStorage may be unavailable when embedded). */ +function readSettingsReturn() { + try { + return sessionStorage.getItem(SETTINGS_RETURN_KEY) === ROUTES.PERSONALITIES ? ROUTES.PERSONALITIES : ROUTES.TALK; + } catch { + return ROUTES.TALK; + } +} + +function storeSettingsReturn(route) { + try { + sessionStorage.setItem(SETTINGS_RETURN_KEY, route); + } catch { + // in-memory fallback is enough + } +} + +/** Honor the desktop app's ?theme=dark|light, overriding prefers-color-scheme. */ +function applyEmbeddedTheme() { + const theme = new URLSearchParams(window.location.search).get("theme"); + if (theme !== "dark" && theme !== "light") return; + document.documentElement.dataset.theme = theme; + document.documentElement.style.colorScheme = theme; +} + +function boot() { + applyEmbeddedTheme(); + + const outlet = $("#view-outlet"); + if (!outlet) { + console.error("#view-outlet missing from index.html"); + return; + } + + const router = createRouter( + { + [ROUTES.TALK]: (ctx) => mountTalkView(ctx), + [ROUTES.PERSONALITIES]: (ctx) => mountHomeView({ ...ctx, navigate: router.navigate }), + [ROUTES.SETTINGS]: (ctx) => mountSettingsView(ctx), + }, + { fallback: ROUTES.TALK, outlet } + ); + + // Settings is an overlay: closing it returns to where it was opened + let settingsReturn = readSettingsReturn(); + const gear = $('[data-action="open-settings"]'); + if (gear) { + gear.addEventListener("click", () => { + if (window.location.hash === ROUTES.SETTINGS) { + router.navigate(settingsReturn); + } else { + settingsReturn = window.location.hash === ROUTES.PERSONALITIES ? ROUTES.PERSONALITIES : ROUTES.TALK; + storeSettingsReturn(settingsReturn); + router.navigate(ROUTES.SETTINGS); + } + }); + } + + const brand = $('[data-action="go-home"]'); + if (brand) { + brand.addEventListener("click", (event) => { + event.preventDefault(); + router.navigate(ROUTES.TALK); + }); + } + + const personalityBadge = $('[data-action="open-personalities"]'); + if (personalityBadge) { + personalityBadge.addEventListener("click", () => router.navigate(ROUTES.PERSONALITIES)); + } + + const back = $('[data-action="go-back"]'); + if (back) { + back.addEventListener("click", () => { + router.navigate(window.location.hash === ROUTES.SETTINGS ? settingsReturn : ROUTES.TALK); + }); + } + + mountPersonalityBadge(document); + + function syncHeaderForRoute() { + const route = window.location.hash || ROUTES.TALK; + if (gear) { + const onSettings = route === ROUTES.SETTINGS; + gear.classList.toggle("is-active", onSettings); + gear.setAttribute("aria-label", onSettings ? "Close settings" : "Open settings"); + } + if (back) { + back.hidden = route === ROUTES.TALK; + const toPersonalities = route === ROUTES.SETTINGS && settingsReturn === ROUTES.PERSONALITIES; + back.setAttribute("aria-label", toPersonalities ? "Back to personalities" : "Back to conversation"); + } + if (route === ROUTES.TALK) showPersonalityBadge(); + else hidePersonalityBadge(); + } + window.addEventListener("hashchange", syncHeaderForRoute); + syncHeaderForRoute(); + + router.start(); +} + +if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", boot, { once: true }); +} else { + boot(); +} diff --git a/src/peachy/static/js/orb.js b/src/peachy/static/js/orb.js new file mode 100644 index 0000000000000000000000000000000000000000..7d739b7f39635ff973a45ac7b161e1522f8e910d --- /dev/null +++ b/src/peachy/static/js/orb.js @@ -0,0 +1,187 @@ +/** + * Conversation orb: SSE-driven state machine → CSS data-state attribute. + * The root is a