import { createHmac, timingSafeEqual } from "node:crypto"; import { existsSync, readFileSync, statSync } from "node:fs"; import { createServer } from "node:http"; import { extname, isAbsolute, relative, resolve, sep } from "node:path"; import { URL } from "node:url"; import { chunkPcm16, DEFAULT_AUDIO_SAMPLE_RATE } from "./audio/pcm.js"; import { HutterRuntime, } from "./hutter/hutterRuntime.js"; import { createMcpHttpHandler } from "./mcp/http.js"; import { createWorkAdventureMcpServer } from "./mcp/server.js"; import { OpenAiRealtimeVoiceManager } from "./realtime/openaiRealtimeVoice.js"; import { createAgentProviderRouter } from "./runtime/providerRouter.js"; import { actionsFromProviderResponse, availableProviderActions } from "./runtime/agentProvider.js"; import { ConnectorQueue } from "./server/connectorQueue.js"; import { PublicAgentRegistry } from "./server/publicAgentRegistry.js"; import { createAgentRoomUrl, renderOpenClawExample, renderPublicTownHome } from "./server/publicTownUi.js"; import { SessionStore } from "./server/sessionStore.js"; import { WorkAdventureAvatarRunner } from "./server/workAdventureAvatarRunner.js"; import { OpenAiTranscriber } from "./stt/openaiTranscription.js"; import { OpenAiSpeechSynthesizer } from "./tts/openaiSpeech.js"; import { workAdventureBridgeScript } from "./workadventure/bridgeScript.js"; import { RemoteConnectorAgentProvider } from "./runtime/providers/remoteConnectorProvider.js"; const host = process.env.WA_AGENT_HOST ?? "127.0.0.1"; const port = Number(process.env.WA_AGENT_PORT ?? 8787); const publicBaseUrl = process.env.WA_AGENT_PUBLIC_URL; const defaultIframeApiUrl = process.env.WA_IFRAME_API_URL ?? "https://play.workadventu.re/iframe_api.js"; const bridgeScriptVersion = process.env.WA_AGENT_BRIDGE_SCRIPT_VERSION ?? "realtime-voice-vad-gated-2"; const bridgeTokenSecret = nonEmptyEnv("WA_AGENT_BRIDGE_SECRET"); const controlToken = nonEmptyEnv("WA_AGENT_CONTROL_TOKEN") ?? bridgeTokenSecret; const connectorToken = nonEmptyEnv("WA_AGENT_CONNECTOR_TOKEN") ?? controlToken; const agentName = process.env.WA_AGENT_NAME ?? "AI Agent"; const observedBridgeEvents = parseObservedEvents(process.env.WA_AGENT_OBSERVED_EVENTS, "hutter:entity,hutter:state,hutter:submission,hutter:message,hutter:lecture"); const sessions = new SessionStore(); const connectors = new ConnectorQueue(); const publicAgents = new PublicAgentRegistry(publicAgentRegistryPath()); const avatarRunner = new WorkAdventureAvatarRunner(); const hutter = new HutterRuntime(); const speech = new OpenAiSpeechSynthesizer(); const enableVoiceReplies = speechConfiguredByEnvironment(); const transcriber = new OpenAiTranscriber(); const voiceMode = normalizeVoiceMode(process.env.WA_AGENT_VOICE_MODE ?? "chained"); const enableVoiceInput = process.env.WA_AGENT_ENABLE_VOICE_INPUT !== "false" && voiceMode !== "off"; const voiceInputConfig = { enabled: enableVoiceInput, mode: voiceMode, sampleRate: Number(process.env.WA_AGENT_VOICE_INPUT_SAMPLE_RATE ?? 24_000), threshold: Number(process.env.WA_AGENT_VOICE_INPUT_THRESHOLD ?? 0.015), silenceMs: Number(process.env.WA_AGENT_VOICE_INPUT_SILENCE_MS ?? 900), minSpeechMs: Number(process.env.WA_AGENT_VOICE_INPUT_MIN_SPEECH_MS ?? 300), maxSpeechMs: Number(process.env.WA_AGENT_VOICE_INPUT_MAX_SPEECH_MS ?? 15_000), cooldownMs: Number(process.env.WA_AGENT_VOICE_INPUT_COOLDOWN_MS ?? 700), realtimeChunkMs: Number(process.env.WA_AGENT_REALTIME_CHUNK_MS ?? 120), realtimeBargeIn: process.env.WA_AGENT_REALTIME_BARGE_IN !== "false", realtimeBargeInThreshold: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_THRESHOLD ?? Math.max(Number(process.env.WA_AGENT_VOICE_INPUT_THRESHOLD ?? 0.015) * 2.5, 0.045)), realtimeBargeInMinMs: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_MIN_MS ?? 700), realtimeBargeInCooldownMs: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_COOLDOWN_MS ?? 1200), realtimeEchoGuardMs: Number(process.env.WA_AGENT_REALTIME_ECHO_GUARD_MS ?? 500), }; const audioOutputConfig = { preRollMs: Number(process.env.WA_AGENT_AUDIO_PRE_ROLL_MS ?? 300), }; const realtimeDefaultVoice = process.env.WA_AGENT_REALTIME_VOICE ?? process.env.OPENAI_REALTIME_VOICE ?? "marin"; const realtimeDefaultInstructions = nonEmptyEnv("WA_AGENT_REALTIME_INSTRUCTIONS"); const realtimeOpenClawAgentId = process.env.OPENCLAW_REALTIME_AGENT_ID ?? process.env.OPENCLAW_AGENT_ID; const realtimeApiKey = nonEmptyEnv("OPENAI_REALTIME_API_KEY") ?? nonEmptyEnv("OPENAI_API_KEY") ?? nonEmptyEnv("OPENAI_TRANSCRIPTION_API_KEY"); const realtimeSessionProfiles = parseRealtimeSessionProfiles(readRealtimeSessionProfilesConfig()); const realtimeVoice = new OpenAiRealtimeVoiceManager({ enabled: voiceMode === "realtime", apiKey: realtimeApiKey, baseUrl: normalizeRealtimeBaseUrl(process.env.OPENAI_REALTIME_BASE_URL ?? "wss://api.openai.com/v1/realtime"), model: process.env.WA_AGENT_REALTIME_MODEL ?? process.env.OPENAI_REALTIME_MODEL ?? "gpt-realtime-2", voice: realtimeDefaultVoice, instructions: realtimeDefaultInstructions ?? defaultRealtimeInstructions(agentName), inputSampleRate: voiceInputConfig.sampleRate, outputSampleRate: Number(process.env.WA_AGENT_REALTIME_OUTPUT_SAMPLE_RATE ?? voiceInputConfig.sampleRate), outputBatchMs: Number(process.env.WA_AGENT_REALTIME_OUTPUT_BATCH_MS ?? 300), vadEagerness: normalizeVadEagerness(process.env.WA_AGENT_REALTIME_VAD_EAGERNESS ?? "high"), audioFloorHoldMs: Number(process.env.WA_AGENT_REALTIME_AUDIO_FLOOR_MS ?? 2500), delegateToOpenClaw: process.env.WA_AGENT_REALTIME_DELEGATE_OPENCLAW !== "false", openClawBin: process.env.OPENCLAW_BIN ?? "openclaw", openClawAgentId: realtimeOpenClawAgentId, openClawProfile: nonEmptyEnv("OPENCLAW_PROFILE"), openClawContainer: nonEmptyEnv("OPENCLAW_CONTAINER"), openClawTimeoutMs: Number(process.env.OPENCLAW_TURN_TIMEOUT_MS ?? 120_000), safetyIdentifier: nonEmptyEnv("OPENAI_SAFETY_IDENTIFIER"), openClawTool: askOpenClawThroughPublicConnector, sessionConfig: (session) => realtimeConfigForSession(session, { profiles: realtimeSessionProfiles, defaultVoice: realtimeDefaultVoice, defaultInstructions: realtimeDefaultInstructions, defaultOpenClawAgentId: realtimeOpenClawAgentId, }), }); const agentProviders = createAgentProviderRouter({ agentName, enableVoiceReplies, connectorQueue: connectors, }); const runtimeClient = createInMemoryRuntimeClient(); const mcpHttpHandler = createMcpHttpHandler(() => createWorkAdventureMcpServer(runtimeClient)); const hutterMapPath = process.env.WA_AGENT_HUTTER_MAP_PATH ?? "/maps/hutter-prize/map.json"; const hutterMapDir = resolveHutterMapDir(); const publicScriptsDir = resolvePublicScriptsDir(); const server = createServer((request, response) => { handleRequest(request, response).catch((error) => { if (error instanceof HttpError) { writeJson(response, error.status, { error: error.message }); return; } console.error(error); writeJson(response, 500, { error: error instanceof Error ? error.message : "Internal server error" }); }); }); server.listen(port, host, () => { console.info(`Agent world runtime listening on http://${host}:${port}`); startPersistedPublicAgentAvatars(); }); async function handleRequest(request, response) { const url = new URL(request.url ?? "/", getRequestBaseUrl(request)); if (request.method === "OPTIONS") { writeCors(response); response.writeHead(204).end(); return; } if (request.method === "GET" && url.pathname === "/health") { writeJson(response, 200, { ok: true, openAiSpeechConfigured: speech.isConfigured(), voiceRepliesEnabled: enableVoiceReplies, voiceMode, voiceInputEnabled: enableVoiceInput, voiceInputConfig, audioOutputConfig, speechToText: transcriber.describe(), realtimeVoice: realtimeVoice.describe(), realtimeVoiceDisabledReason: getRealtimeVoiceDisabledReason() ?? null, realtimeSessionProfiles: { count: realtimeSessionProfiles.size, ids: Array.from(realtimeSessionProfiles.keys()), }, agentProviders: agentProviders.describe(), connectors: connectors.summary(), publicAgents: publicAgents.summary(), avatars: avatarRunner.summary(), hutter: { results: hutter.getState().leaderboard.length, messages: hutter.getState().recentMessages.length, }, }); return; } if (request.method === "GET" && url.pathname === "/") { const baseUrl = getRuntimeBaseUrl(request); writeText(response, 200, renderPublicTownHome({ baseUrl, mapPath: hutterMapPath, }), "text/html; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/examples/openclaw") { writeText(response, 200, renderOpenClawExample(getRuntimeBaseUrl(request)), "text/markdown; charset=utf-8"); return; } if (url.pathname === "/public/agents" || url.pathname.startsWith("/public/agents/")) { await handlePublicAgentsRoute(request, response, url); return; } if (request.method === "GET" && url.pathname.startsWith("/maps/hutter-prize/")) { serveHutterMapAsset(response, url.pathname); return; } if (request.method === "GET" && url.pathname.startsWith("/scripts/")) { servePublicScript(response, url.pathname); return; } if (request.method === "GET" && url.pathname === "/sessions") { writeJson(response, 200, { sessions: sessions.list() }); return; } if (url.pathname === "/mcp") { writeCors(response); assertControlToken(request); const parsedBody = request.method === "POST" ? await readJson(request) : undefined; await mcpHttpHandler.handle(request, response, parsedBody); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge.js") { writeText(response, 200, workAdventureBridgeScript, "text/javascript; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge.html") { const sessionId = requireSessionId(url); assertBridgeToken(url, sessionId); if (!controlToken) { publicAgents.ensureSessionPresence(sessionId); } publicAgents.markSessionSeen(sessionId); const iframeApiUrl = url.searchParams.get("waApi") ?? defaultIframeApiUrl; writeText(response, 200, renderBridgeHtml(sessionId, iframeApiUrl, getRuntimeBaseUrl(request)), "text/html; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge-url") { const sessionId = requireSessionId(url); assertBridgeUrlAccess(request, url, sessionId); if (!controlToken) { publicAgents.ensureSessionPresence(sessionId); } publicAgents.markSessionSeen(sessionId); const iframeApiUrl = url.searchParams.get("waApi") ?? defaultIframeApiUrl; writeJson(response, 200, { url: createBridgeUrl({ sessionId, iframeApiUrl }, getRuntimeBaseUrl(request)) }); return; } if (request.method === "GET" && url.pathname === "/connectors") { assertControlToken(request); writeJson(response, 200, { connectors: connectors.summary() }); return; } const connectorRoute = matchConnectorRoute(url.pathname); if (connectorRoute) { await handleConnectorRoute(request, response, url, connectorRoute.connectorId, connectorRoute.action); return; } if (url.pathname === "/hutter" || url.pathname.startsWith("/hutter/")) { await handleHutterRoute(request, response, url); return; } const sessionRoute = matchSessionRoute(url.pathname); if (sessionRoute) { const session = sessions.getOrCreate(sessionRoute.sessionId); await handleSessionRoute(request, response, url, sessionRoute.action, session); return; } writeJson(response, 404, { error: "Not found" }); } async function handleConnectorRoute(request, response, url, connectorId, action) { if (!connectorToken) { publicAgents.ensureConnectorPresence(connectorId); } assertConnectorToken(request, connectorId); publicAgents.markConnectorSeen(connectorId); if (request.method === "GET" && action === "observations") { const after = Number(url.searchParams.get("after") ?? 0); const timeoutMs = Math.min(30_000, Math.max(0, Number(url.searchParams.get("timeoutMs") ?? 25_000))); const deliveries = timeoutMs > 0 ? await connectors.waitForDeliveries(connectorId, after, timeoutMs) : connectors.getDeliveriesAfter(connectorId, after); writeJson(response, 200, { cursor: deliveries.reduce((cursor, delivery) => Math.max(cursor, delivery.id), after), deliveries, }); return; } if (request.method === "POST" && action === "results") { const body = (await readJson(request)); const deliveryId = typeof body.deliveryId === "number" ? body.deliveryId : undefined; const delivery = deliveryId !== undefined ? connectors.getDelivery(connectorId, deliveryId) : undefined; const sessionId = body.sessionId ?? delivery?.session.id; if (!sessionId) { writeJson(response, 400, { error: "Missing sessionId" }); return; } const session = sessions.getOrCreate(sessionId); const suppressSessionActions = isRecord(delivery?.metadata) && delivery.metadata.suppressSessionActions === true; if (body.error) { console.error(`Connector "${connectorId}" failed delivery ${deliveryId ?? "unknown"}: ${body.error}`); if (delivery?.visibleErrors && !suppressSessionActions) { session.enqueue({ type: "say", channel: "local", text: `${delivery.displayName ?? delivery.providerId} is unavailable: ${body.error}`, }); } } else { const actionSource = body.actions ?? body.response; const actions = actionsFromProviderResponse(actionSource, { defaultChannel: "bubble" }); if (!suppressSessionActions) { for (const worldAction of actions) { session.enqueue(worldAction); } if (delivery?.enableVoiceReplies && enableVoiceReplies) { for (const worldAction of actions) { if (worldAction.type === "say") { await enqueueSpeech(session, worldAction.text); } } } } } if (body.stopTyping !== false && !suppressSessionActions) { session.enqueue({ type: "stopTyping", channel: "bubble" }); } if (deliveryId !== undefined) { connectors.resolveDeliveryResult(connectorId, deliveryId, body); } writeJson(response, 202, { ok: true }); return; } writeJson(response, 404, { error: "Not found" }); } async function handlePublicAgentsRoute(request, response, url) { if (request.method === "GET" && url.pathname === "/public/agents") { writeJson(response, 200, { agents: publicAgents.list() }); return; } if (request.method === "POST" && url.pathname === "/public/agents") { const registration = publicAgents.register((await readJson(request))); const baseUrl = getRuntimeBaseUrl(request); const avatarUrl = createAgentRoomUrl(registration, baseUrl, hutterMapPath); avatarRunner.ensure(registration.agent, avatarUrl); writeJson(response, 201, { ...registration, avatarUrl, connector: { gatewayUrl: baseUrl, connectorId: registration.agent.connectorId, tokenEnv: "WA_AGENT_CONNECTOR_TOKEN", }, }); return; } const match = /^\/public\/agents\/([^/]+)$/.exec(url.pathname); if (request.method === "GET" && match) { const agent = publicAgents.getById(decodeURIComponent(match[1])); if (!agent) { writeJson(response, 404, { error: "Unknown public agent" }); return; } writeJson(response, 200, { agent }); return; } if (request.method === "DELETE" && match) { const agentId = decodeURIComponent(match[1]); const agent = publicAgents.getInternalById(agentId); if (!agent) { writeJson(response, 404, { error: "Unknown public agent" }); return; } const bearerToken = getControlToken(request); const avatarToken = url.searchParams.get("agentKey") ?? url.searchParams.get("avatarToken"); const authorized = isControlTokenValid(request) || publicAgents.authenticateConnector(agent.connectorId, bearerToken) || publicAgents.authenticateAvatar(agent.sessionId, avatarToken ?? bearerToken); if (!authorized) { writeJson(response, 401, { error: "Missing or invalid agent token" }); return; } avatarRunner.stop(agent.sessionId); publicAgents.deleteById(agentId); writeJson(response, 202, { ok: true, deleted: agentId }); return; } writeJson(response, 404, { error: "Not found" }); } async function handleHutterRoute(request, response, url) { if (request.method === "GET" && (url.pathname === "/hutter" || url.pathname === "/hutter/state")) { writeJson(response, 200, hutter.getState()); return; } if (request.method === "GET" && url.pathname === "/hutter/lecture") { writeJson(response, 200, hutter.getLecture()); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/start") { const lecture = hutter.startLecture((await readJson(request))); writeJson(response, 201, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/next") { const lecture = hutter.nextLectureSlide(); writeJson(response, 200, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/stop") { const lecture = hutter.stopLecture(); writeJson(response, 200, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/transcript") { const lecture = hutter.addLectureTranscript((await readJson(request))); writeJson(response, 201, { lecture, state: hutter.getState() }); return; } if (request.method === "GET" && url.pathname === "/hutter/leaderboard") { writeJson(response, 200, { leaderboard: hutter.getState().leaderboard }); return; } const entityMatch = /^\/hutter\/entities\/([^/]+)$/.exec(url.pathname); if (request.method === "GET" && entityMatch) { const entity = hutter.getEntity(decodeURIComponent(entityMatch[1])); if (!entity) { writeJson(response, 404, { error: "Unknown Hutter entity" }); return; } writeJson(response, 200, { entity, state: hutter.getState(), }); return; } if (request.method === "POST" && url.pathname === "/hutter/results") { const result = hutter.submitResult((await readJson(request))); writeJson(response, 201, { result, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/messages") { const message = hutter.postMessage((await readJson(request))); writeJson(response, 201, { message, state: hutter.getState() }); return; } writeJson(response, 404, { error: "Not found" }); } async function handleSessionRoute(request, response, url, action, session) { if (request.method === "GET" && action === "") { writeJson(response, 200, { session: session.summary(), recentObservations: session.recentObservations(), }); return; } if (request.method === "POST" && action === "observations") { const observation = (await readJson(request)); await dispatchObservation(session, observation); writeJson(response, 202, { ok: true }); return; } if (request.method === "GET" && action === "actions") { publicAgents.markSessionSeen(session.id); const after = Number(url.searchParams.get("after") ?? 0); const timeoutMs = Math.min(30_000, Math.max(0, Number(url.searchParams.get("timeoutMs") ?? 25_000))); const actions = timeoutMs > 0 ? await session.waitForActions(after, timeoutMs) : session.getActionsAfter(after); writeJson(response, 200, { cursor: actions.reduce((cursor, worldAction) => Math.max(cursor, worldAction.id), after), actions, }); return; } if (request.method === "POST" && action === "actions") { const body = (await readJson(request)); if (!body.action) { writeJson(response, 400, { error: "Missing action" }); return; } const queued = session.enqueue(body.action); writeJson(response, 202, { action: queued }); return; } if (request.method === "POST" && action === "say") { const body = (await readJson(request)); if (!body.text) { writeJson(response, 400, { error: "Missing text" }); return; } const queued = session.enqueue({ type: "say", channel: body.channel ?? "bubble", text: body.text }); if (body.voice === true) { await enqueueSpeech(session, body.text); } writeJson(response, 202, { action: queued }); return; } if (request.method === "POST" && action === "speak") { const body = (await readJson(request)); if (!body.text) { writeJson(response, 400, { error: "Missing text" }); return; } await enqueueSpeech(session, body.text); writeJson(response, 202, { ok: true }); return; } if (request.method === "POST" && action === "agent-messages") { const body = (await readJson(request)); if (!body.fromSessionId) { writeJson(response, 400, { error: "Missing fromSessionId" }); return; } if (!body.message) { writeJson(response, 400, { error: "Missing message" }); return; } await dispatchObservation(session, { type: "agentMessage", fromSessionId: body.fromSessionId, message: body.message, data: body.data, }); writeJson(response, 202, { ok: true }); return; } if (request.method === "POST" && action === "realtime/start") { const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } writeJson(response, 202, { ok: true, realtime: await realtimeVoice.start(session) }); return; } if (request.method === "POST" && action === "realtime/audio") { const body = (await readJson(request)); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } const sampleRate = typeof body.sampleRate === "number" ? body.sampleRate : NaN; if (!Number.isFinite(sampleRate) || typeof body.pcm16Base64 !== "string" || body.pcm16Base64.length === 0) { writeJson(response, 400, { error: "Missing sampleRate or pcm16Base64" }); return; } const pcmBytes = Buffer.from(body.pcm16Base64, "base64").byteLength; const realtime = await realtimeVoice.appendAudio(session, { sampleRate, pcm16Base64: body.pcm16Base64 }); session.addObservation({ type: "realtimeVoiceInput", status: "server-accepted", sampleRate, durationMs: finiteOptionalNumber(body.durationMs), speechMs: finiteOptionalNumber(body.speechMs), totalMs: finiteOptionalNumber(body.totalMs), peakRms: finiteOptionalNumber(body.peakRms), reason: body.reason, bytes: pcmBytes, }); const committed = body.reason === "vad" || body.reason === "stop" ? await realtimeVoice.commitAudio(session, body.reason).catch((error) => { session.addObservation({ type: "bridgeError", message: "Realtime audio commit failed", details: { value: error instanceof Error ? error.message : String(error) }, }); return undefined; }) : undefined; writeJson(response, 202, { ok: true, realtime: committed ?? realtime }); return; } if (request.method === "POST" && action === "realtime/message") { const body = (await readJson(request)); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } if (typeof body.text !== "string" || body.text.trim().length === 0) { writeJson(response, 400, { error: "Missing text" }); return; } writeJson(response, 202, { ok: true, realtime: await realtimeVoice.sendText(session, body.text.trim()) }); return; } if (request.method === "POST" && action === "realtime/interrupt") { const body = (await readJson(request).catch(() => ({}))); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } writeJson(response, 202, { ok: true, realtime: realtimeVoice.interrupt(session, body.reason) }); return; } if (request.method === "POST" && action === "realtime/stop") { writeJson(response, 202, { ok: true, realtime: realtimeVoice.stop(session.id) }); return; } if (request.method === "POST" && action === "voice-utterances") { const body = (await readJson(request)); if (!enableVoiceInput) { writeJson(response, 202, { ok: true, ignored: "voice input disabled" }); return; } if (!transcriber.isConfigured()) { session.addObservation({ type: "bridgeError", message: "Voice input unavailable", details: { value: "OPENAI_API_KEY or OPENAI_TRANSCRIPTION_API_KEY is not configured for transcription." }, }); writeJson(response, 202, { ok: false, ignored: "transcriber not configured" }); return; } const sampleRate = typeof body.sampleRate === "number" ? body.sampleRate : NaN; if (!Number.isFinite(sampleRate) || typeof body.pcm16Base64 !== "string" || body.pcm16Base64.length === 0) { writeJson(response, 400, { error: "Missing sampleRate or pcm16Base64" }); return; } const pcm16 = Buffer.from(body.pcm16Base64, "base64"); const transcript = await transcriber.transcribePcm16({ pcm16: new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength), sampleRate, }); if (!transcript) { writeJson(response, 202, { ok: true, transcript: "", ignored: "empty transcript" }); return; } await dispatchObservation(session, { type: "voiceTranscript", transcript, durationMs: Number.isFinite(body.durationMs) ? body.durationMs : undefined, sampleRate, source: body.source ?? "proximityMeeting", }); writeJson(response, 202, { ok: true, transcript }); return; } writeJson(response, 404, { error: "Not found" }); } function createInMemoryRuntimeClient() { return { async listSessions() { return sessions.list(); }, async getSession(sessionId) { const session = sessions.getOrCreate(sessionId); return { session: session.summary(), recentObservations: session.recentObservations(), }; }, async enqueueAction(sessionId, action) { return { action: sessions.getOrCreate(sessionId).enqueue(action) }; }, async say(sessionId, text, options = {}) { const session = sessions.getOrCreate(sessionId); const action = session.enqueue({ type: "say", channel: options.channel ?? "bubble", text }); if (options.voice ?? false) { await enqueueSpeech(session, text); } return { action }; }, async speak(sessionId, text) { await enqueueSpeech(sessions.getOrCreate(sessionId), text); return { ok: true }; }, async sendAgentMessage(targetSessionId, fromSessionId, message, data) { const session = sessions.getOrCreate(targetSessionId); await dispatchObservation(session, { type: "agentMessage", fromSessionId, message, data, }); return { ok: true, targetSessionId }; }, async createBridgeUrl(options) { return createBridgeUrl(options, publicBaseUrl ?? `http://${host}:${port}`); }, async getHutterState() { return hutter.getState(); }, async getHutterEntity(entityId) { const entity = hutter.getEntity(entityId); if (!entity) { throw new Error(`Unknown Hutter entity: ${entityId}`); } return { entity, state: hutter.getState(), }; }, async submitHutterResult(input) { return hutter.submitResult(input); }, async postHutterMessage(input) { return hutter.postMessage(input); }, async getHutterLecture() { return hutter.getLecture(); }, async startHutterLecture(input) { return hutter.startLecture(input); }, async nextHutterLectureSlide() { return hutter.nextLectureSlide(); }, async stopHutterLecture() { return hutter.stopLecture(); }, async addHutterLectureTranscript(input) { return hutter.addLectureTranscript(input); }, }; } async function dispatchObservation(session, observation) { session.addObservation(observation); const provider = resolveAgentProvider(session.id); await provider.handleObservation({ agentName, session: session.summary(), observation, recentObservations: session.recentObservations(), }, (worldAction) => session.enqueue(worldAction), (text) => enqueueSpeech(session, text)); } function resolveAgentProvider(sessionId) { const publicAgent = publicAgents.getInternalBySessionId(sessionId); if (!publicAgent) { return agentProviders.resolve(sessionId); } return new RemoteConnectorAgentProvider({ id: publicAgent.connectorId, connectorId: publicAgent.connectorId, connectorQueue: connectors, agentId: publicAgent.agentId, displayName: publicAgent.displayName, sessionPrefix: "public", respondToAllBubbleMessages: true, invokeOnEvents: false, visibleErrors: true, enableVoiceReplies, publicRole: publicAgent.role, publicDescription: publicAgent.description, }); } async function askOpenClawThroughPublicConnector(session, request, config) { const publicAgent = publicAgents.getInternalBySessionId(session.id); if (!publicAgent) { return undefined; } const delivery = connectors.enqueue(publicAgent.connectorId, { providerId: `${publicAgent.connectorId}:realtime-tool`, agentId: publicAgent.agentId, displayName: publicAgent.displayName, session: session.summary(), workAdventureSessionId: session.id, openClawSessionId: `public-realtime:${publicAgent.connectorId}:${session.id}`, message: [ "You are being consulted by an OpenAI Realtime voice layer inside WorkAdventure.", "Return a concise answer suitable to be spoken aloud. Do not include markdown unless it is essential.", `Voice agent: ${config.displayName} (${session.id}).`, `Public agent role: ${publicAgent.role}.`, "", `User request: ${request}`, ].join("\n"), observation: { type: "agentMessage", fromSessionId: "openai-realtime", message: request, data: { source: "realtime-tool" }, }, recentObservations: session.recentObservations(), availableActions: Array.from(availableProviderActions), enableVoiceReplies: false, visibleErrors: false, metadata: { source: "realtime-tool", suppressSessionActions: true, }, }); const timeoutMs = Number(process.env.WA_AGENT_REALTIME_TOOL_TIMEOUT_MS ?? 4_000); const result = await connectors.waitForDeliveryResult(publicAgent.connectorId, delivery.id, timeoutMs); if (!result) { return { ok: false, answer: [ `OpenClaw did not answer quickly for ${config.displayName}.`, "Continue now from the live WorkAdventure conversation instead of waiting for the tool.", "Keep the spoken reply concise and useful.", ].join(" "), }; } if (result.error) { return { ok: false, answer: [ `OpenClaw is currently unavailable for ${config.displayName}.`, "Continue now from the live WorkAdventure conversation instead of waiting for the tool.", "Keep the spoken reply concise and useful.", ].join(" "), }; } const answer = connectorResultText(result); return { ok: true, answer: answer || "OpenClaw completed the request but returned no spoken text.", }; } function connectorResultText(result) { const actionSource = result.actions ?? result.response; const actions = actionsFromProviderResponse(actionSource, { defaultChannel: "bubble" }); const spoken = actions .filter((action) => action.type === "say") .map((action) => action.text.trim()) .filter(Boolean); if (spoken.length > 0) { return spoken.join("\n"); } return extractTextValue(actionSource); } function extractTextValue(value) { if (typeof value === "string") { return value.trim() || undefined; } if (!isRecord(value)) { return undefined; } for (const key of ["text", "message", "content", "answer", "output"]) { const entry = value[key]; if (typeof entry === "string" && entry.trim()) { return entry.trim(); } } return undefined; } async function enqueueSpeech(session, text) { const pcm = await speech.synthesizePcm(text); for (const chunk of chunkPcm16(pcm, DEFAULT_AUDIO_SAMPLE_RATE)) { session.enqueue({ type: "speakPcm", sampleRate: chunk.sampleRate, pcm16Base64: chunk.pcm16Base64, }); } session.enqueue({ type: "closeAudioStream" }); } function createBridgeUrl(options, base) { const bridgeUrl = new URL("/workadventure/bridge.html", base); bridgeUrl.searchParams.set("session", options.sessionId); bridgeUrl.searchParams.set("waApi", options.iframeApiUrl ?? defaultIframeApiUrl); bridgeUrl.searchParams.set("v", bridgeScriptVersion); const token = createBridgeToken(options.sessionId); if (token) { bridgeUrl.searchParams.set("token", token); } return bridgeUrl.toString(); } function renderBridgeHtml(sessionId, iframeApiUrl, runtimeBaseUrl) { const bridgeScriptUrl = new URL("/workadventure/bridge.js", runtimeBaseUrl); bridgeScriptUrl.searchParams.set("v", bridgeScriptVersion); const initialActionCursor = process.env.WA_AGENT_ACTION_INITIAL_CURSOR === "latest" ? sessions.getOrCreate(sessionId).summary().actionsCount : Number(process.env.WA_AGENT_ACTION_INITIAL_CURSOR ?? 0); return `