import { createHmac, timingSafeEqual } from "node:crypto"; import { existsSync, readFileSync, statSync } from "node:fs"; import { createServer } from "node:http"; import { extname, isAbsolute, relative, resolve, sep } from "node:path"; import { URL } from "node:url"; import { chunkPcm16, DEFAULT_AUDIO_SAMPLE_RATE } from "./audio/pcm.js"; import { HutterRuntime, } from "./hutter/hutterRuntime.js"; import { createMcpHttpHandler } from "./mcp/http.js"; import { createWorkAdventureMcpServer } from "./mcp/server.js"; import { OpenAiRealtimeVoiceManager } from "./realtime/openaiRealtimeVoice.js"; import { createAgentProviderRouter } from "./runtime/providerRouter.js"; import { actionsFromProviderResponse, availableProviderActions } from "./runtime/agentProvider.js"; import { ConnectorQueue } from "./server/connectorQueue.js"; import { PublicAgentRegistry } from "./server/publicAgentRegistry.js"; import { createAgentRoomUrl, renderOpenClawExample, renderPublicTownHome } from "./server/publicTownUi.js"; import { SessionStore } from "./server/sessionStore.js"; import { WorkAdventureAvatarRunner } from "./server/workAdventureAvatarRunner.js"; import { OpenAiTranscriber } from "./stt/openaiTranscription.js"; import { OpenAiSpeechSynthesizer } from "./tts/openaiSpeech.js"; import { workAdventureBridgeScript } from "./workadventure/bridgeScript.js"; import { RemoteConnectorAgentProvider } from "./runtime/providers/remoteConnectorProvider.js"; const host = process.env.WA_AGENT_HOST ?? "127.0.0.1"; const port = Number(process.env.WA_AGENT_PORT ?? 8787); const publicBaseUrl = process.env.WA_AGENT_PUBLIC_URL; const defaultIframeApiUrl = process.env.WA_IFRAME_API_URL ?? "https://play.workadventu.re/iframe_api.js"; const bridgeScriptVersion = process.env.WA_AGENT_BRIDGE_SCRIPT_VERSION ?? "realtime-voice-vad-gated-2"; const bridgeTokenSecret = nonEmptyEnv("WA_AGENT_BRIDGE_SECRET"); const controlToken = nonEmptyEnv("WA_AGENT_CONTROL_TOKEN") ?? bridgeTokenSecret; const connectorToken = nonEmptyEnv("WA_AGENT_CONNECTOR_TOKEN") ?? controlToken; const agentName = process.env.WA_AGENT_NAME ?? "AI Agent"; const observedBridgeEvents = parseObservedEvents(process.env.WA_AGENT_OBSERVED_EVENTS, "hutter:entity,hutter:state,hutter:submission,hutter:message,hutter:lecture"); const sessions = new SessionStore(); const connectors = new ConnectorQueue(); const publicAgents = new PublicAgentRegistry(publicAgentRegistryPath()); const avatarRunner = new WorkAdventureAvatarRunner(); const hutter = new HutterRuntime(); const speech = new OpenAiSpeechSynthesizer(); const enableVoiceReplies = speechConfiguredByEnvironment(); const transcriber = new OpenAiTranscriber(); const voiceMode = normalizeVoiceMode(process.env.WA_AGENT_VOICE_MODE ?? "chained"); const enableVoiceInput = process.env.WA_AGENT_ENABLE_VOICE_INPUT !== "false" && voiceMode !== "off"; const voiceInputConfig = { enabled: enableVoiceInput, mode: voiceMode, sampleRate: Number(process.env.WA_AGENT_VOICE_INPUT_SAMPLE_RATE ?? 24_000), threshold: Number(process.env.WA_AGENT_VOICE_INPUT_THRESHOLD ?? 0.015), silenceMs: Number(process.env.WA_AGENT_VOICE_INPUT_SILENCE_MS ?? 900), minSpeechMs: Number(process.env.WA_AGENT_VOICE_INPUT_MIN_SPEECH_MS ?? 300), maxSpeechMs: Number(process.env.WA_AGENT_VOICE_INPUT_MAX_SPEECH_MS ?? 15_000), cooldownMs: Number(process.env.WA_AGENT_VOICE_INPUT_COOLDOWN_MS ?? 700), realtimeChunkMs: Number(process.env.WA_AGENT_REALTIME_CHUNK_MS ?? 120), realtimeBargeIn: process.env.WA_AGENT_REALTIME_BARGE_IN !== "false", realtimeBargeInThreshold: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_THRESHOLD ?? Math.max(Number(process.env.WA_AGENT_VOICE_INPUT_THRESHOLD ?? 0.015) * 2.5, 0.045)), realtimeBargeInMinMs: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_MIN_MS ?? 700), realtimeBargeInCooldownMs: Number(process.env.WA_AGENT_REALTIME_BARGE_IN_COOLDOWN_MS ?? 1200), realtimeEchoGuardMs: Number(process.env.WA_AGENT_REALTIME_ECHO_GUARD_MS ?? 500), }; const audioOutputConfig = { preRollMs: Number(process.env.WA_AGENT_AUDIO_PRE_ROLL_MS ?? 300), }; const realtimeDefaultVoice = process.env.WA_AGENT_REALTIME_VOICE ?? process.env.OPENAI_REALTIME_VOICE ?? "marin"; const realtimeDefaultInstructions = nonEmptyEnv("WA_AGENT_REALTIME_INSTRUCTIONS"); const realtimeOpenClawAgentId = process.env.OPENCLAW_REALTIME_AGENT_ID ?? process.env.OPENCLAW_AGENT_ID; const realtimeApiKey = nonEmptyEnv("OPENAI_REALTIME_API_KEY") ?? nonEmptyEnv("OPENAI_API_KEY") ?? nonEmptyEnv("OPENAI_TRANSCRIPTION_API_KEY"); const realtimeSessionProfiles = parseRealtimeSessionProfiles(readRealtimeSessionProfilesConfig()); const realtimeVoice = new OpenAiRealtimeVoiceManager({ enabled: voiceMode === "realtime", apiKey: realtimeApiKey, baseUrl: normalizeRealtimeBaseUrl(process.env.OPENAI_REALTIME_BASE_URL ?? "wss://api.openai.com/v1/realtime"), model: process.env.WA_AGENT_REALTIME_MODEL ?? process.env.OPENAI_REALTIME_MODEL ?? "gpt-realtime-2", voice: realtimeDefaultVoice, instructions: realtimeDefaultInstructions ?? defaultRealtimeInstructions(agentName), inputSampleRate: voiceInputConfig.sampleRate, outputSampleRate: Number(process.env.WA_AGENT_REALTIME_OUTPUT_SAMPLE_RATE ?? voiceInputConfig.sampleRate), outputBatchMs: Number(process.env.WA_AGENT_REALTIME_OUTPUT_BATCH_MS ?? 300), vadEagerness: normalizeVadEagerness(process.env.WA_AGENT_REALTIME_VAD_EAGERNESS ?? "high"), audioFloorHoldMs: Number(process.env.WA_AGENT_REALTIME_AUDIO_FLOOR_MS ?? 2500), delegateToOpenClaw: process.env.WA_AGENT_REALTIME_DELEGATE_OPENCLAW !== "false", openClawBin: process.env.OPENCLAW_BIN ?? "openclaw", openClawAgentId: realtimeOpenClawAgentId, openClawProfile: nonEmptyEnv("OPENCLAW_PROFILE"), openClawContainer: nonEmptyEnv("OPENCLAW_CONTAINER"), openClawTimeoutMs: Number(process.env.OPENCLAW_TURN_TIMEOUT_MS ?? 120_000), safetyIdentifier: nonEmptyEnv("OPENAI_SAFETY_IDENTIFIER"), openClawTool: askOpenClawThroughPublicConnector, sessionConfig: (session) => realtimeConfigForSession(session, { profiles: realtimeSessionProfiles, defaultVoice: realtimeDefaultVoice, defaultInstructions: realtimeDefaultInstructions, defaultOpenClawAgentId: realtimeOpenClawAgentId, }), }); const agentProviders = createAgentProviderRouter({ agentName, enableVoiceReplies, connectorQueue: connectors, }); const runtimeClient = createInMemoryRuntimeClient(); const mcpHttpHandler = createMcpHttpHandler(() => createWorkAdventureMcpServer(runtimeClient)); const hutterMapPath = process.env.WA_AGENT_HUTTER_MAP_PATH ?? "/maps/hutter-prize/map.json"; const hutterMapDir = resolveHutterMapDir(); const publicScriptsDir = resolvePublicScriptsDir(); const server = createServer((request, response) => { handleRequest(request, response).catch((error) => { if (error instanceof HttpError) { writeJson(response, error.status, { error: error.message }); return; } console.error(error); writeJson(response, 500, { error: error instanceof Error ? error.message : "Internal server error" }); }); }); server.listen(port, host, () => { console.info(`Agent world runtime listening on http://${host}:${port}`); startPersistedPublicAgentAvatars(); }); async function handleRequest(request, response) { const url = new URL(request.url ?? "/", getRequestBaseUrl(request)); if (request.method === "OPTIONS") { writeCors(response); response.writeHead(204).end(); return; } if (request.method === "GET" && url.pathname === "/health") { writeJson(response, 200, { ok: true, openAiSpeechConfigured: speech.isConfigured(), voiceRepliesEnabled: enableVoiceReplies, voiceMode, voiceInputEnabled: enableVoiceInput, voiceInputConfig, audioOutputConfig, speechToText: transcriber.describe(), realtimeVoice: realtimeVoice.describe(), realtimeVoiceDisabledReason: getRealtimeVoiceDisabledReason() ?? null, realtimeSessionProfiles: { count: realtimeSessionProfiles.size, ids: Array.from(realtimeSessionProfiles.keys()), }, agentProviders: agentProviders.describe(), connectors: connectors.summary(), publicAgents: publicAgents.summary(), avatars: avatarRunner.summary(), hutter: { results: hutter.getState().leaderboard.length, messages: hutter.getState().recentMessages.length, }, }); return; } if (request.method === "GET" && url.pathname === "/") { const baseUrl = getRuntimeBaseUrl(request); writeText(response, 200, renderPublicTownHome({ baseUrl, mapPath: hutterMapPath, }), "text/html; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/examples/openclaw") { writeText(response, 200, renderOpenClawExample(getRuntimeBaseUrl(request)), "text/markdown; charset=utf-8"); return; } if (url.pathname === "/public/agents" || url.pathname.startsWith("/public/agents/")) { await handlePublicAgentsRoute(request, response, url); return; } if (request.method === "GET" && url.pathname.startsWith("/maps/hutter-prize/")) { serveHutterMapAsset(response, url.pathname); return; } if (request.method === "GET" && url.pathname.startsWith("/scripts/")) { servePublicScript(response, url.pathname); return; } if (request.method === "GET" && url.pathname === "/sessions") { writeJson(response, 200, { sessions: sessions.list() }); return; } if (url.pathname === "/mcp") { writeCors(response); assertControlToken(request); const parsedBody = request.method === "POST" ? await readJson(request) : undefined; await mcpHttpHandler.handle(request, response, parsedBody); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge.js") { writeText(response, 200, workAdventureBridgeScript, "text/javascript; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge.html") { const sessionId = requireSessionId(url); assertBridgeToken(url, sessionId); if (!controlToken) { publicAgents.ensureSessionPresence(sessionId); } publicAgents.markSessionSeen(sessionId); const iframeApiUrl = url.searchParams.get("waApi") ?? defaultIframeApiUrl; writeText(response, 200, renderBridgeHtml(sessionId, iframeApiUrl, getRuntimeBaseUrl(request)), "text/html; charset=utf-8"); return; } if (request.method === "GET" && url.pathname === "/workadventure/bridge-url") { const sessionId = requireSessionId(url); assertBridgeUrlAccess(request, url, sessionId); if (!controlToken) { publicAgents.ensureSessionPresence(sessionId); } publicAgents.markSessionSeen(sessionId); const iframeApiUrl = url.searchParams.get("waApi") ?? defaultIframeApiUrl; writeJson(response, 200, { url: createBridgeUrl({ sessionId, iframeApiUrl }, getRuntimeBaseUrl(request)) }); return; } if (request.method === "GET" && url.pathname === "/connectors") { assertControlToken(request); writeJson(response, 200, { connectors: connectors.summary() }); return; } const connectorRoute = matchConnectorRoute(url.pathname); if (connectorRoute) { await handleConnectorRoute(request, response, url, connectorRoute.connectorId, connectorRoute.action); return; } if (url.pathname === "/hutter" || url.pathname.startsWith("/hutter/")) { await handleHutterRoute(request, response, url); return; } const sessionRoute = matchSessionRoute(url.pathname); if (sessionRoute) { const session = sessions.getOrCreate(sessionRoute.sessionId); await handleSessionRoute(request, response, url, sessionRoute.action, session); return; } writeJson(response, 404, { error: "Not found" }); } async function handleConnectorRoute(request, response, url, connectorId, action) { if (!connectorToken) { publicAgents.ensureConnectorPresence(connectorId); } assertConnectorToken(request, connectorId); publicAgents.markConnectorSeen(connectorId); if (request.method === "GET" && action === "observations") { const after = Number(url.searchParams.get("after") ?? 0); const timeoutMs = Math.min(30_000, Math.max(0, Number(url.searchParams.get("timeoutMs") ?? 25_000))); const deliveries = timeoutMs > 0 ? await connectors.waitForDeliveries(connectorId, after, timeoutMs) : connectors.getDeliveriesAfter(connectorId, after); writeJson(response, 200, { cursor: deliveries.reduce((cursor, delivery) => Math.max(cursor, delivery.id), after), deliveries, }); return; } if (request.method === "POST" && action === "results") { const body = (await readJson(request)); const deliveryId = typeof body.deliveryId === "number" ? body.deliveryId : undefined; const delivery = deliveryId !== undefined ? connectors.getDelivery(connectorId, deliveryId) : undefined; const sessionId = body.sessionId ?? delivery?.session.id; if (!sessionId) { writeJson(response, 400, { error: "Missing sessionId" }); return; } const session = sessions.getOrCreate(sessionId); const suppressSessionActions = isRecord(delivery?.metadata) && delivery.metadata.suppressSessionActions === true; if (body.error) { console.error(`Connector "${connectorId}" failed delivery ${deliveryId ?? "unknown"}: ${body.error}`); if (delivery?.visibleErrors && !suppressSessionActions) { session.enqueue({ type: "say", channel: "local", text: `${delivery.displayName ?? delivery.providerId} is unavailable: ${body.error}`, }); } } else { const actionSource = body.actions ?? body.response; const actions = actionsFromProviderResponse(actionSource, { defaultChannel: "bubble" }); if (!suppressSessionActions) { for (const worldAction of actions) { session.enqueue(worldAction); } if (delivery?.enableVoiceReplies && enableVoiceReplies) { for (const worldAction of actions) { if (worldAction.type === "say") { await enqueueSpeech(session, worldAction.text); } } } } } if (body.stopTyping !== false && !suppressSessionActions) { session.enqueue({ type: "stopTyping", channel: "bubble" }); } if (deliveryId !== undefined) { connectors.resolveDeliveryResult(connectorId, deliveryId, body); } writeJson(response, 202, { ok: true }); return; } writeJson(response, 404, { error: "Not found" }); } async function handlePublicAgentsRoute(request, response, url) { if (request.method === "GET" && url.pathname === "/public/agents") { writeJson(response, 200, { agents: publicAgents.list() }); return; } if (request.method === "POST" && url.pathname === "/public/agents") { const registration = publicAgents.register((await readJson(request))); const baseUrl = getRuntimeBaseUrl(request); const avatarUrl = createAgentRoomUrl(registration, baseUrl, hutterMapPath); avatarRunner.ensure(registration.agent, avatarUrl); writeJson(response, 201, { ...registration, avatarUrl, connector: { gatewayUrl: baseUrl, connectorId: registration.agent.connectorId, tokenEnv: "WA_AGENT_CONNECTOR_TOKEN", }, }); return; } const match = /^\/public\/agents\/([^/]+)$/.exec(url.pathname); if (request.method === "GET" && match) { const agent = publicAgents.getById(decodeURIComponent(match[1])); if (!agent) { writeJson(response, 404, { error: "Unknown public agent" }); return; } writeJson(response, 200, { agent }); return; } if (request.method === "DELETE" && match) { const agentId = decodeURIComponent(match[1]); const agent = publicAgents.getInternalById(agentId); if (!agent) { writeJson(response, 404, { error: "Unknown public agent" }); return; } const bearerToken = getControlToken(request); const avatarToken = url.searchParams.get("agentKey") ?? url.searchParams.get("avatarToken"); const authorized = isControlTokenValid(request) || publicAgents.authenticateConnector(agent.connectorId, bearerToken) || publicAgents.authenticateAvatar(agent.sessionId, avatarToken ?? bearerToken); if (!authorized) { writeJson(response, 401, { error: "Missing or invalid agent token" }); return; } avatarRunner.stop(agent.sessionId); publicAgents.deleteById(agentId); writeJson(response, 202, { ok: true, deleted: agentId }); return; } writeJson(response, 404, { error: "Not found" }); } async function handleHutterRoute(request, response, url) { if (request.method === "GET" && (url.pathname === "/hutter" || url.pathname === "/hutter/state")) { writeJson(response, 200, hutter.getState()); return; } if (request.method === "GET" && url.pathname === "/hutter/lecture") { writeJson(response, 200, hutter.getLecture()); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/start") { const lecture = hutter.startLecture((await readJson(request))); writeJson(response, 201, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/next") { const lecture = hutter.nextLectureSlide(); writeJson(response, 200, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/stop") { const lecture = hutter.stopLecture(); writeJson(response, 200, { lecture, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/lecture/transcript") { const lecture = hutter.addLectureTranscript((await readJson(request))); writeJson(response, 201, { lecture, state: hutter.getState() }); return; } if (request.method === "GET" && url.pathname === "/hutter/leaderboard") { writeJson(response, 200, { leaderboard: hutter.getState().leaderboard }); return; } const entityMatch = /^\/hutter\/entities\/([^/]+)$/.exec(url.pathname); if (request.method === "GET" && entityMatch) { const entity = hutter.getEntity(decodeURIComponent(entityMatch[1])); if (!entity) { writeJson(response, 404, { error: "Unknown Hutter entity" }); return; } writeJson(response, 200, { entity, state: hutter.getState(), }); return; } if (request.method === "POST" && url.pathname === "/hutter/results") { const result = hutter.submitResult((await readJson(request))); writeJson(response, 201, { result, state: hutter.getState() }); return; } if (request.method === "POST" && url.pathname === "/hutter/messages") { const message = hutter.postMessage((await readJson(request))); writeJson(response, 201, { message, state: hutter.getState() }); return; } writeJson(response, 404, { error: "Not found" }); } async function handleSessionRoute(request, response, url, action, session) { if (request.method === "GET" && action === "") { writeJson(response, 200, { session: session.summary(), recentObservations: session.recentObservations(), }); return; } if (request.method === "POST" && action === "observations") { const observation = (await readJson(request)); await dispatchObservation(session, observation); writeJson(response, 202, { ok: true }); return; } if (request.method === "GET" && action === "actions") { publicAgents.markSessionSeen(session.id); const after = Number(url.searchParams.get("after") ?? 0); const timeoutMs = Math.min(30_000, Math.max(0, Number(url.searchParams.get("timeoutMs") ?? 25_000))); const actions = timeoutMs > 0 ? await session.waitForActions(after, timeoutMs) : session.getActionsAfter(after); writeJson(response, 200, { cursor: actions.reduce((cursor, worldAction) => Math.max(cursor, worldAction.id), after), actions, }); return; } if (request.method === "POST" && action === "actions") { const body = (await readJson(request)); if (!body.action) { writeJson(response, 400, { error: "Missing action" }); return; } const queued = session.enqueue(body.action); writeJson(response, 202, { action: queued }); return; } if (request.method === "POST" && action === "say") { const body = (await readJson(request)); if (!body.text) { writeJson(response, 400, { error: "Missing text" }); return; } const queued = session.enqueue({ type: "say", channel: body.channel ?? "bubble", text: body.text }); if (body.voice === true) { await enqueueSpeech(session, body.text); } writeJson(response, 202, { action: queued }); return; } if (request.method === "POST" && action === "speak") { const body = (await readJson(request)); if (!body.text) { writeJson(response, 400, { error: "Missing text" }); return; } await enqueueSpeech(session, body.text); writeJson(response, 202, { ok: true }); return; } if (request.method === "POST" && action === "agent-messages") { const body = (await readJson(request)); if (!body.fromSessionId) { writeJson(response, 400, { error: "Missing fromSessionId" }); return; } if (!body.message) { writeJson(response, 400, { error: "Missing message" }); return; } await dispatchObservation(session, { type: "agentMessage", fromSessionId: body.fromSessionId, message: body.message, data: body.data, }); writeJson(response, 202, { ok: true }); return; } if (request.method === "POST" && action === "realtime/start") { const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } writeJson(response, 202, { ok: true, realtime: await realtimeVoice.start(session) }); return; } if (request.method === "POST" && action === "realtime/audio") { const body = (await readJson(request)); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } const sampleRate = typeof body.sampleRate === "number" ? body.sampleRate : NaN; if (!Number.isFinite(sampleRate) || typeof body.pcm16Base64 !== "string" || body.pcm16Base64.length === 0) { writeJson(response, 400, { error: "Missing sampleRate or pcm16Base64" }); return; } const pcmBytes = Buffer.from(body.pcm16Base64, "base64").byteLength; const realtime = await realtimeVoice.appendAudio(session, { sampleRate, pcm16Base64: body.pcm16Base64 }); session.addObservation({ type: "realtimeVoiceInput", status: "server-accepted", sampleRate, durationMs: finiteOptionalNumber(body.durationMs), speechMs: finiteOptionalNumber(body.speechMs), totalMs: finiteOptionalNumber(body.totalMs), peakRms: finiteOptionalNumber(body.peakRms), reason: body.reason, bytes: pcmBytes, }); const committed = body.reason === "vad" || body.reason === "stop" ? await realtimeVoice.commitAudio(session, body.reason).catch((error) => { session.addObservation({ type: "bridgeError", message: "Realtime audio commit failed", details: { value: error instanceof Error ? error.message : String(error) }, }); return undefined; }) : undefined; writeJson(response, 202, { ok: true, realtime: committed ?? realtime }); return; } if (request.method === "POST" && action === "realtime/message") { const body = (await readJson(request)); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } if (typeof body.text !== "string" || body.text.trim().length === 0) { writeJson(response, 400, { error: "Missing text" }); return; } writeJson(response, 202, { ok: true, realtime: await realtimeVoice.sendText(session, body.text.trim()) }); return; } if (request.method === "POST" && action === "realtime/interrupt") { const body = (await readJson(request).catch(() => ({}))); const disabledReason = getRealtimeVoiceDisabledReason(); if (disabledReason) { writeJson(response, 202, { ok: true, ignored: disabledReason }); return; } writeJson(response, 202, { ok: true, realtime: realtimeVoice.interrupt(session, body.reason) }); return; } if (request.method === "POST" && action === "realtime/stop") { writeJson(response, 202, { ok: true, realtime: realtimeVoice.stop(session.id) }); return; } if (request.method === "POST" && action === "voice-utterances") { const body = (await readJson(request)); if (!enableVoiceInput) { writeJson(response, 202, { ok: true, ignored: "voice input disabled" }); return; } if (!transcriber.isConfigured()) { session.addObservation({ type: "bridgeError", message: "Voice input unavailable", details: { value: "OPENAI_API_KEY or OPENAI_TRANSCRIPTION_API_KEY is not configured for transcription." }, }); writeJson(response, 202, { ok: false, ignored: "transcriber not configured" }); return; } const sampleRate = typeof body.sampleRate === "number" ? body.sampleRate : NaN; if (!Number.isFinite(sampleRate) || typeof body.pcm16Base64 !== "string" || body.pcm16Base64.length === 0) { writeJson(response, 400, { error: "Missing sampleRate or pcm16Base64" }); return; } const pcm16 = Buffer.from(body.pcm16Base64, "base64"); const transcript = await transcriber.transcribePcm16({ pcm16: new Uint8Array(pcm16.buffer, pcm16.byteOffset, pcm16.byteLength), sampleRate, }); if (!transcript) { writeJson(response, 202, { ok: true, transcript: "", ignored: "empty transcript" }); return; } await dispatchObservation(session, { type: "voiceTranscript", transcript, durationMs: Number.isFinite(body.durationMs) ? body.durationMs : undefined, sampleRate, source: body.source ?? "proximityMeeting", }); writeJson(response, 202, { ok: true, transcript }); return; } writeJson(response, 404, { error: "Not found" }); } function createInMemoryRuntimeClient() { return { async listSessions() { return sessions.list(); }, async getSession(sessionId) { const session = sessions.getOrCreate(sessionId); return { session: session.summary(), recentObservations: session.recentObservations(), }; }, async enqueueAction(sessionId, action) { return { action: sessions.getOrCreate(sessionId).enqueue(action) }; }, async say(sessionId, text, options = {}) { const session = sessions.getOrCreate(sessionId); const action = session.enqueue({ type: "say", channel: options.channel ?? "bubble", text }); if (options.voice ?? false) { await enqueueSpeech(session, text); } return { action }; }, async speak(sessionId, text) { await enqueueSpeech(sessions.getOrCreate(sessionId), text); return { ok: true }; }, async sendAgentMessage(targetSessionId, fromSessionId, message, data) { const session = sessions.getOrCreate(targetSessionId); await dispatchObservation(session, { type: "agentMessage", fromSessionId, message, data, }); return { ok: true, targetSessionId }; }, async createBridgeUrl(options) { return createBridgeUrl(options, publicBaseUrl ?? `http://${host}:${port}`); }, async getHutterState() { return hutter.getState(); }, async getHutterEntity(entityId) { const entity = hutter.getEntity(entityId); if (!entity) { throw new Error(`Unknown Hutter entity: ${entityId}`); } return { entity, state: hutter.getState(), }; }, async submitHutterResult(input) { return hutter.submitResult(input); }, async postHutterMessage(input) { return hutter.postMessage(input); }, async getHutterLecture() { return hutter.getLecture(); }, async startHutterLecture(input) { return hutter.startLecture(input); }, async nextHutterLectureSlide() { return hutter.nextLectureSlide(); }, async stopHutterLecture() { return hutter.stopLecture(); }, async addHutterLectureTranscript(input) { return hutter.addLectureTranscript(input); }, }; } async function dispatchObservation(session, observation) { session.addObservation(observation); const provider = resolveAgentProvider(session.id); await provider.handleObservation({ agentName, session: session.summary(), observation, recentObservations: session.recentObservations(), }, (worldAction) => session.enqueue(worldAction), (text) => enqueueSpeech(session, text)); } function resolveAgentProvider(sessionId) { const publicAgent = publicAgents.getInternalBySessionId(sessionId); if (!publicAgent) { return agentProviders.resolve(sessionId); } return new RemoteConnectorAgentProvider({ id: publicAgent.connectorId, connectorId: publicAgent.connectorId, connectorQueue: connectors, agentId: publicAgent.agentId, displayName: publicAgent.displayName, sessionPrefix: "public", respondToAllBubbleMessages: true, invokeOnEvents: false, visibleErrors: true, enableVoiceReplies, publicRole: publicAgent.role, publicDescription: publicAgent.description, }); } async function askOpenClawThroughPublicConnector(session, request, config) { const publicAgent = publicAgents.getInternalBySessionId(session.id); if (!publicAgent) { return undefined; } const delivery = connectors.enqueue(publicAgent.connectorId, { providerId: `${publicAgent.connectorId}:realtime-tool`, agentId: publicAgent.agentId, displayName: publicAgent.displayName, session: session.summary(), workAdventureSessionId: session.id, openClawSessionId: `public-realtime:${publicAgent.connectorId}:${session.id}`, message: [ "You are being consulted by an OpenAI Realtime voice layer inside WorkAdventure.", "Return a concise answer suitable to be spoken aloud. Do not include markdown unless it is essential.", `Voice agent: ${config.displayName} (${session.id}).`, `Public agent role: ${publicAgent.role}.`, "", `User request: ${request}`, ].join("\n"), observation: { type: "agentMessage", fromSessionId: "openai-realtime", message: request, data: { source: "realtime-tool" }, }, recentObservations: session.recentObservations(), availableActions: Array.from(availableProviderActions), enableVoiceReplies: false, visibleErrors: false, metadata: { source: "realtime-tool", suppressSessionActions: true, }, }); const timeoutMs = Number(process.env.WA_AGENT_REALTIME_TOOL_TIMEOUT_MS ?? 4_000); const result = await connectors.waitForDeliveryResult(publicAgent.connectorId, delivery.id, timeoutMs); if (!result) { return { ok: false, answer: [ `OpenClaw did not answer quickly for ${config.displayName}.`, "Continue now from the live WorkAdventure conversation instead of waiting for the tool.", "Keep the spoken reply concise and useful.", ].join(" "), }; } if (result.error) { return { ok: false, answer: [ `OpenClaw is currently unavailable for ${config.displayName}.`, "Continue now from the live WorkAdventure conversation instead of waiting for the tool.", "Keep the spoken reply concise and useful.", ].join(" "), }; } const answer = connectorResultText(result); return { ok: true, answer: answer || "OpenClaw completed the request but returned no spoken text.", }; } function connectorResultText(result) { const actionSource = result.actions ?? result.response; const actions = actionsFromProviderResponse(actionSource, { defaultChannel: "bubble" }); const spoken = actions .filter((action) => action.type === "say") .map((action) => action.text.trim()) .filter(Boolean); if (spoken.length > 0) { return spoken.join("\n"); } return extractTextValue(actionSource); } function extractTextValue(value) { if (typeof value === "string") { return value.trim() || undefined; } if (!isRecord(value)) { return undefined; } for (const key of ["text", "message", "content", "answer", "output"]) { const entry = value[key]; if (typeof entry === "string" && entry.trim()) { return entry.trim(); } } return undefined; } async function enqueueSpeech(session, text) { const pcm = await speech.synthesizePcm(text); for (const chunk of chunkPcm16(pcm, DEFAULT_AUDIO_SAMPLE_RATE)) { session.enqueue({ type: "speakPcm", sampleRate: chunk.sampleRate, pcm16Base64: chunk.pcm16Base64, }); } session.enqueue({ type: "closeAudioStream" }); } function createBridgeUrl(options, base) { const bridgeUrl = new URL("/workadventure/bridge.html", base); bridgeUrl.searchParams.set("session", options.sessionId); bridgeUrl.searchParams.set("waApi", options.iframeApiUrl ?? defaultIframeApiUrl); bridgeUrl.searchParams.set("v", bridgeScriptVersion); const token = createBridgeToken(options.sessionId); if (token) { bridgeUrl.searchParams.set("token", token); } return bridgeUrl.toString(); } function renderBridgeHtml(sessionId, iframeApiUrl, runtimeBaseUrl) { const bridgeScriptUrl = new URL("/workadventure/bridge.js", runtimeBaseUrl); bridgeScriptUrl.searchParams.set("v", bridgeScriptVersion); const initialActionCursor = process.env.WA_AGENT_ACTION_INITIAL_CURSOR === "latest" ? sessions.getOrCreate(sessionId).summary().actionsCount : Number(process.env.WA_AGENT_ACTION_INITIAL_CURSOR ?? 0); return ` WorkAdventure AI Agent Bridge `; } function knownAgentDisplayNames() { const names = new Set(); names.add(agentName); for (const agent of publicAgents.listInternal()) { names.add(agent.displayName); names.add(agent.id); names.add(agent.connectorId); } for (const name of ["Ada", "Grace", "Shannon", "Alan", "Lin", "Dave"]) { names.add(name); names.add(`agent-${name.toLowerCase()}`); } return Array.from(names).filter((name) => name.trim().length > 0); } function bridgeVoiceInputConfig() { if (voiceMode !== "realtime" || !getRealtimeVoiceDisabledReason()) { return voiceInputConfig; } return { ...voiceInputConfig, enabled: false, }; } function getRealtimeVoiceDisabledReason() { if (voiceMode !== "realtime" || !enableVoiceInput) { return "realtime voice disabled"; } if (!realtimeApiKey) { return "realtime voice missing OpenAI API key"; } return undefined; } function matchSessionRoute(pathname) { const match = /^\/sessions\/([^/]+)\/?(.*)$/.exec(pathname); if (!match) { return undefined; } return { sessionId: decodeURIComponent(match[1]), action: match[2] ?? "", }; } function matchConnectorRoute(pathname) { const match = /^\/connectors\/([^/]+)\/?(.*)$/.exec(pathname); if (!match) { return undefined; } return { connectorId: decodeURIComponent(match[1]), action: match[2] ?? "", }; } function requireSessionId(url) { const sessionId = url.searchParams.get("session"); if (!sessionId) { throw new HttpError(400, "Missing session"); } return sessionId; } function createBridgeToken(sessionId) { if (!bridgeTokenSecret) { return undefined; } return createHmac("sha256", bridgeTokenSecret).update(sessionId).digest("hex"); } function assertBridgeToken(url, sessionId) { if (!bridgeTokenSecret) { return; } const expected = createBridgeToken(sessionId); const actual = url.searchParams.get("token"); if (!expected || !actual) { throw new HttpError(401, "Missing bridge token"); } const expectedBuffer = Buffer.from(expected); const actualBuffer = Buffer.from(actual); if (expectedBuffer.byteLength !== actualBuffer.byteLength || !timingSafeEqual(expectedBuffer, actualBuffer)) { throw new HttpError(401, "Invalid bridge token"); } } function assertControlToken(request) { if (!controlToken) { return; } const actual = getControlToken(request); if (!actual || !timingSafeStringEqual(controlToken, actual)) { throw new HttpError(401, "Missing or invalid control token"); } } function assertBridgeUrlAccess(request, url, sessionId) { if (!controlToken) { return; } if (isControlTokenValid(request)) { return; } const agentKey = url.searchParams.get("agentKey") ?? url.searchParams.get("avatarToken"); if (publicAgents.authenticateAvatar(sessionId, agentKey ?? undefined)) { return; } throw new HttpError(401, "Missing or invalid control token"); } function assertConnectorToken(request, connectorId) { const actual = getControlToken(request); if (!connectorToken && !publicAgents.getInternalByConnectorId(connectorId)) { return; } if (!connectorToken && publicAgents.allowsTokenlessConnector(connectorId) && !actual) { return; } if (connectorToken && actual && timingSafeStringEqual(connectorToken, actual)) { return; } if (publicAgents.authenticateConnector(connectorId, actual)) { return; } if (!actual) { throw new HttpError(401, "Missing or invalid connector token"); } throw new HttpError(401, "Missing or invalid connector token"); } function isControlTokenValid(request) { const actual = getControlToken(request); return Boolean(controlToken && actual && timingSafeStringEqual(controlToken, actual)); } function getControlToken(request) { const authorization = headerValue(request.headers.authorization); const bearerMatch = authorization?.match(/^Bearer\s+(.+)$/i); if (bearerMatch) { return bearerMatch[1]; } return headerValue(request.headers["x-wa-agent-control-token"]); } function timingSafeStringEqual(expected, actual) { const expectedBuffer = Buffer.from(expected); const actualBuffer = Buffer.from(actual); return expectedBuffer.byteLength === actualBuffer.byteLength && timingSafeEqual(expectedBuffer, actualBuffer); } function headerValue(value) { return Array.isArray(value) ? value[0] : value; } function speechConfiguredByEnvironment() { return speech.isConfigured() && process.env.WA_AGENT_ENABLE_VOICE !== "false"; } function nonEmptyEnv(name) { const value = process.env[name]?.trim(); return value ? value : undefined; } function finiteOptionalNumber(value) { return typeof value === "number" && Number.isFinite(value) ? value : undefined; } function parseObservedEvents(value, fallback) { return (value ?? fallback) .split(",") .map((entry) => entry.trim()) .filter((entry) => entry.length > 0); } function readRealtimeSessionProfilesConfig() { const inlineConfig = process.env.WA_AGENT_REALTIME_VOICES_JSON ?? process.env.WA_AGENT_REALTIME_SESSIONS_JSON; if (inlineConfig) { return inlineConfig; } const configPath = nonEmptyEnv("WA_AGENT_REALTIME_VOICES_FILE") ?? nonEmptyEnv("WA_AGENT_REALTIME_SESSIONS_FILE"); if (!configPath) { return undefined; } const resolvedPath = isAbsolute(configPath) ? configPath : resolve(process.env.INIT_CWD ?? process.cwd(), configPath); return readFileSync(resolvedPath, "utf8"); } function parseRealtimeSessionProfiles(value) { const profiles = new Map(); if (!value) { return profiles; } let parsed; try { parsed = JSON.parse(value); } catch (error) { throw new Error(`WA_AGENT_REALTIME_VOICES_JSON must be valid JSON: ${error instanceof Error ? error.message : String(error)}`); } if (!isRecord(parsed)) { throw new Error("WA_AGENT_REALTIME_VOICES_JSON must be a JSON object keyed by session id or session prefix"); } for (const [id, profile] of Object.entries(parsed)) { if (typeof profile === "string") { profiles.set(id, { voice: nonEmptyString(profile) }); continue; } if (!isRecord(profile)) { continue; } profiles.set(id, { displayName: nonEmptyString(profile.displayName), name: nonEmptyString(profile.name), voice: nonEmptyString(profile.voice), instructions: nonEmptyString(profile.instructions), openClawAgentId: nonEmptyString(profile.openClawAgentId), agentId: nonEmptyString(profile.agentId), }); } return profiles; } function realtimeConfigForSession(session, context) { const routeId = getSessionRouteId(session.id); const profile = context.profiles.get(session.id) ?? (routeId ? context.profiles.get(routeId) : undefined); const displayName = profile?.displayName ?? profile?.name ?? routeId ?? session.id; const openClawAgentId = profile?.openClawAgentId ?? profile?.agentId ?? context.defaultOpenClawAgentId; return { displayName, voice: profile?.voice ?? context.defaultVoice, instructions: profile?.instructions ?? context.defaultInstructions ?? defaultRealtimeInstructions(displayName), openClawAgentId, }; } function getSessionRouteId(sessionId) { const separatorIndex = sessionId.indexOf(":"); if (separatorIndex <= 0) { return undefined; } return sessionId.slice(0, separatorIndex); } function nonEmptyString(value) { if (typeof value !== "string") { return undefined; } const trimmed = value.trim(); return trimmed.length > 0 ? trimmed : undefined; } function normalizeVoiceMode(value) { const normalized = value.trim().toLowerCase(); if (normalized === "off" || normalized === "false" || normalized === "none") { return "off"; } if (normalized === "realtime" || normalized === "openai-realtime") { return "realtime"; } return "chained"; } function normalizeVadEagerness(value) { const normalized = value.trim().toLowerCase(); return normalized === "low" || normalized === "medium" || normalized === "high" || normalized === "auto" ? normalized : "high"; } function normalizeRealtimeBaseUrl(value) { const normalized = value.trim().replace(/\/$/, ""); if (normalized.startsWith("https://")) { return `wss://${normalized.slice("https://".length)}`; } if (normalized.startsWith("http://")) { return `ws://${normalized.slice("http://".length)}`; } return normalized; } function defaultRealtimeInstructions(name) { return [ `You are ${name}, a realtime voice presence inside Auto Research Town, a WorkAdventure lab for the Hutter Prize.`, "Your permanent mission is the Hutter Prize: help reduce the enwik8 score, measured as compressed archive bytes plus a self-contained decompressor.", "Keep spoken replies concise, natural, low-latency, and explicitly tied to Hutter Prize work.", "Talk about leaderboard evidence, graph trends, compression methods, decompressor-size tradeoffs, reproducible experiments, and submission strategy.", "If a human or another agent drifts into generic chat, politely steer the discussion back to the Hutter Prize.", "Do not invent benchmark results. Ask for evidence, propose a test, or describe a concrete compression experiment when uncertain.", "For repository work, implementation details, or anything that needs the connected OpenClaw project agent, call ask_openclaw and then summarize the Hutter-relevant result aloud.", "Do not print chat messages. This voice session is independent from WorkAdventure text chat.", ].join("\n"); } function resolveHutterMapDir() { const configured = nonEmptyEnv("WA_AGENT_HUTTER_MAP_DIR"); if (configured) { return isAbsolute(configured) ? configured : resolve(process.cwd(), configured); } const roots = [process.env.INIT_CWD, process.cwd(), resolve(process.cwd(), "../..")].filter(Boolean); for (const root of roots) { const candidate = resolve(root, "maps/hutter-prize"); if (existsSync(candidate)) { return candidate; } } return resolve(process.cwd(), "maps/hutter-prize"); } function publicAgentRegistryPath() { const configured = nonEmptyEnv("WA_AGENT_PUBLIC_AGENTS_FILE"); if (configured) { return isAbsolute(configured) ? configured : resolve(process.cwd(), configured); } if (existsSync("/data")) { return "/data/public-agents.json"; } return undefined; } function resolvePublicScriptsDir() { const configured = nonEmptyEnv("WA_AGENT_PUBLIC_SCRIPTS_DIR"); if (configured) { return isAbsolute(configured) ? configured : resolve(process.cwd(), configured); } const roots = [process.env.INIT_CWD, process.cwd(), resolve(process.cwd(), "../..")].filter(Boolean); for (const root of roots) { for (const candidate of [resolve(root, "scripts"), resolve(root, "libs/agent-world-runtime/examples")]) { if (existsSync(candidate)) { return candidate; } } } return resolve(process.cwd(), "scripts"); } function serveHutterMapAsset(response, pathname) { const routePrefix = "/maps/hutter-prize/"; const relativePath = decodeURIComponent(pathname.slice(routePrefix.length)) || "map.json"; const resolvedPath = resolve(hutterMapDir, relativePath); const relativeToRoot = relative(hutterMapDir, resolvedPath); if (relativeToRoot.startsWith("..") || relativeToRoot.includes(`..${sep}`) || isAbsolute(relativeToRoot)) { writeJson(response, 403, { error: "Forbidden" }); return; } if (!existsSync(resolvedPath) || !statSync(resolvedPath).isFile()) { writeJson(response, 404, { error: "Map asset not found" }); return; } writeCors(response); response.writeHead(200, { "Content-Type": contentTypeForPath(resolvedPath), ...noStoreHeaders(), }); response.end(readFileSync(resolvedPath)); } function servePublicScript(response, pathname) { const routePrefix = "/scripts/"; const relativePath = decodeURIComponent(pathname.slice(routePrefix.length)); const resolvedPath = resolve(publicScriptsDir, relativePath); const relativeToRoot = relative(publicScriptsDir, resolvedPath); if (relativeToRoot.startsWith("..") || relativeToRoot.includes(`..${sep}`) || isAbsolute(relativeToRoot)) { writeJson(response, 403, { error: "Forbidden" }); return; } if (!existsSync(resolvedPath) || !statSync(resolvedPath).isFile()) { writeJson(response, 404, { error: "Script not found" }); return; } writeCors(response); response.writeHead(200, { "Content-Type": contentTypeForPath(resolvedPath), ...noStoreHeaders(), }); response.end(readFileSync(resolvedPath)); } function noStoreHeaders() { return { "Cache-Control": "no-store, max-age=0", Pragma: "no-cache", Expires: "0", }; } function contentTypeForPath(pathname) { switch (extname(pathname).toLowerCase()) { case ".json": case ".tmj": return "application/json; charset=utf-8"; case ".js": case ".mjs": return "text/javascript; charset=utf-8"; case ".html": return "text/html; charset=utf-8"; case ".css": return "text/css; charset=utf-8"; case ".png": return "image/png"; case ".jpg": case ".jpeg": return "image/jpeg"; case ".svg": return "image/svg+xml"; default: return "application/octet-stream"; } } async function readJson(request) { const chunks = []; for await (const chunk of request) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } if (chunks.length === 0) { return {}; } return JSON.parse(Buffer.concat(chunks).toString("utf-8")); } function getRuntimeBaseUrl(request) { return publicBaseUrl ?? getRequestBaseUrl(request); } function startPersistedPublicAgentAvatars() { if (!publicBaseUrl) { return; } for (const agent of publicAgents.listInternal()) { avatarRunner.ensure(agent, createAgentRoomUrl({ agent, avatarToken: agent.avatarToken }, publicBaseUrl, hutterMapPath)); } } function getRequestBaseUrl(request) { const proto = request.headers["x-forwarded-proto"] ?? "http"; const hostHeader = request.headers["x-forwarded-host"] ?? request.headers.host ?? `${host}:${port}`; return `${Array.isArray(proto) ? proto[0] : proto}://${Array.isArray(hostHeader) ? hostHeader[0] : hostHeader}`; } function writeJson(response, status, value) { writeCors(response); response.writeHead(status, { "Content-Type": "application/json; charset=utf-8" }); response.end(JSON.stringify(value)); } function writeText(response, status, value, contentType) { writeCors(response); response.writeHead(status, { "Content-Type": contentType }); response.end(value); } function writeCors(response) { response.setHeader("Access-Control-Allow-Origin", "*"); response.setHeader("Access-Control-Allow-Methods", "GET,POST,DELETE,OPTIONS"); response.setHeader("Access-Control-Allow-Headers", "Content-Type,Authorization,X-WA-Agent-Control-Token,mcp-session-id,MCP-Protocol-Version,Last-Event-ID"); response.setHeader("Access-Control-Expose-Headers", "mcp-session-id,MCP-Protocol-Version"); } function escapeHtml(value) { return value.replaceAll("&", "&").replaceAll('"', """).replaceAll("<", "<").replaceAll(">", ">"); } function isRecord(value) { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } class HttpError extends Error { status; constructor(status, message) { super(message); this.status = status; } } //# sourceMappingURL=server.js.map