diff --git a/src/audit/audit-config.test.ts b/src/audit/audit-config.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..af85a66bb95b8ac4539fcc29820bdb2798104b2f --- /dev/null +++ b/src/audit/audit-config.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { + isAuditLedgerEnabled, + isExecutionIdentityCollectionEnabled, + resolveAuditMessageMode, +} from "./audit-config.js"; + +describe("isAuditLedgerEnabled", () => { + it("defaults to enabled without config or audit section", () => { + expect(isAuditLedgerEnabled(undefined)).toBe(true); + expect(isAuditLedgerEnabled({})).toBe(true); + expect(isAuditLedgerEnabled({ logging: { audit: {} } })).toBe(true); + }); + + it("stays enabled on explicit true", () => { + expect(isAuditLedgerEnabled({ logging: { audit: { enabled: true } } })).toBe(true); + }); + + it("disables only on explicit false", () => { + expect(isAuditLedgerEnabled({ logging: { audit: { enabled: false } } })).toBe(false); + }); + + it("keeps message metadata off until explicitly enabled", () => { + expect(resolveAuditMessageMode(undefined)).toBe("off"); + expect(resolveAuditMessageMode({ logging: { audit: {} } })).toBe("off"); + expect(resolveAuditMessageMode({ logging: { audit: { messages: "direct" } } })).toBe("direct"); + expect(resolveAuditMessageMode({ logging: { audit: { messages: "all" } } })).toBe("all"); + }); + + it("keeps execution identity off until both switches are explicitly enabled", () => { + expect(isExecutionIdentityCollectionEnabled(undefined)).toBe(false); + expect(isExecutionIdentityCollectionEnabled({ logging: { audit: {} } })).toBe(false); + expect( + isExecutionIdentityCollectionEnabled({ + logging: { audit: { enabled: true, executionIdentity: true } }, + }), + ).toBe(true); + expect( + isExecutionIdentityCollectionEnabled({ + logging: { audit: { enabled: false, executionIdentity: true } }, + }), + ).toBe(false); + }); +}); diff --git a/src/audit/audit-config.ts b/src/audit/audit-config.ts new file mode 100644 index 0000000000000000000000000000000000000000..f67657eaa4ac6318476e43653c737a02f60ac955 --- /dev/null +++ b/src/audit/audit-config.ts @@ -0,0 +1,23 @@ +/** Resolves whether the metadata-only audit ledger records new events. */ +import type { OpenClawConfig } from "../config/types.openclaw.js"; + +export type AuditMessageMode = "off" | "direct" | "all"; + +/** + * The ledger is on by default: an audit trail enabled only after an incident + * cannot explain the incident. `logging.audit.enabled: false` stops new event inserts after + * restart; audit queries still serve retained rows until they expire. + */ +export function isAuditLedgerEnabled(cfg: OpenClawConfig | undefined): boolean { + return cfg?.logging?.audit?.enabled !== false; +} + +/** Execution identity is retained only after an explicit startup-scoped opt-in. */ +export function isExecutionIdentityCollectionEnabled(cfg: OpenClawConfig | undefined): boolean { + return isAuditLedgerEnabled(cfg) && cfg?.logging?.audit?.executionIdentity === true; +} + +/** Message metadata remains an explicit opt-in inside the default-on ledger. */ +export function resolveAuditMessageMode(cfg: OpenClawConfig | undefined): AuditMessageMode { + return cfg?.logging?.audit?.messages ?? "off"; +} diff --git a/src/audit/audit-event-store.message.test.ts b/src/audit/audit-event-store.message.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d0159bce4feca4be2588f103fde48bdb973780b4 --- /dev/null +++ b/src/audit/audit-event-store.message.test.ts @@ -0,0 +1,465 @@ +import { afterAll, afterEach, describe, expect, it } from "vitest"; +import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { listAuditEvents, recordAuditEvent } from "./audit-event-store.js"; +import type { AuditEventInput, MessageAuditEventInput } from "./audit-event-types.js"; + +const tempDirs: string[] = []; +const AUDIT_REF_RE = /^hmac-sha256:v1:[a-f0-9]{32}:[a-f0-9]{64}$/u; + +type AgentRunAuditEventInput = Extract; +type InboundMessageAuditEventInput = Extract; +type OutboundMessageAuditEventInput = Extract; +type OutboundMessageAuditTerminalInput = Extract< + OutboundMessageAuditEventInput, + { action: "message.outbound.finished" } +>; +type OutboundTerminalFields = + | "deliveryKind" + | "errorCode" + | "failureStage" + | "outcome" + | "reasonCode" + | "status"; +type OutboundMessageAuditTerminal = { + [Status in OutboundMessageAuditTerminalInput["status"]]: Pick< + Extract, + OutboundTerminalFields + >; +}[OutboundMessageAuditTerminalInput["status"]]; + +function createDatabaseOptions() { + return { env: { OPENCLAW_STATE_DIR: makeTempDir(tempDirs, "openclaw-message-audit-") } }; +} + +type InboundMessageOverrides = Partial< + Pick< + InboundMessageAuditEventInput, + | "sourceId" + | "sourceSequence" + | "occurredAt" + | "actorId" + | "agentId" + | "runId" + | "channel" + | "conversationKind" + | "durationMs" + | "resultCount" + | "accountId" + | "conversationId" + | "messageId" + | "targetId" + > +>; + +function messageInput(overrides: InboundMessageOverrides = {}): InboundMessageAuditEventInput { + return { + sourceId: "message-source-1", + sourceSequence: 1, + occurredAt: Date.now(), + kind: "message", + action: "message.inbound.processed", + status: "succeeded", + actorType: "channel_sender", + actorId: "sender:+15551234567", + agentId: "main", + runId: "run-1", + direction: "inbound", + channel: "telegram", + conversationKind: "direct", + outcome: "completed", + durationMs: 12, + resultCount: 1, + accountId: "operator@example.test", + conversationId: "chat-123", + messageId: "message-456", + targetId: "target-789", + ...overrides, + }; +} + +type OutboundMessageOverrides = Partial< + Pick< + OutboundMessageAuditEventInput, + | "sourceId" + | "sourceSequence" + | "occurredAt" + | "channel" + | "conversationKind" + | "durationMs" + | "resultCount" + | "accountId" + | "conversationId" + | "messageId" + | "targetId" + > +>; + +function outboundMessageInput( + terminal: OutboundMessageAuditTerminal, + overrides: OutboundMessageOverrides = {}, +): OutboundMessageAuditEventInput { + return { + sourceId: "outbound-message-source-1", + sourceSequence: 1, + occurredAt: Date.now(), + kind: "message", + action: "message.outbound.finished", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "run-1", + direction: "outbound", + channel: "telegram", + conversationKind: "direct", + durationMs: 12, + resultCount: 1, + accountId: "operator@example.test", + conversationId: "chat-123", + messageId: "message-456", + targetId: "target-789", + ...terminal, + ...overrides, + }; +} + +function runInput( + overrides: Partial< + Pick + > = {}, +): AgentRunAuditEventInput { + return { + sourceId: "run-source-1", + sourceSequence: 1, + occurredAt: Date.now(), + kind: "agent_run", + action: "agent.run.started", + status: "started", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "run-1", + ...overrides, + }; +} + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +afterAll(() => { + cleanupTempDirs(tempDirs); +}); + +describe("message audit persistence", () => { + it("stores stable domain-separated pseudonyms without raw transport identities", () => { + const database = createDatabaseOptions(); + const rawIdentity = "same-raw-identity"; + const first = recordAuditEvent( + messageInput({ + actorId: rawIdentity, + accountId: rawIdentity, + conversationId: rawIdentity, + messageId: rawIdentity, + targetId: rawIdentity, + }), + database, + ); + + expect(first).toMatchObject({ + schemaVersion: 1, + kind: "message", + direction: "inbound", + channel: "telegram", + conversationKind: "direct", + outcome: "completed", + durationMs: 12, + resultCount: 1, + redaction: "metadata_only", + }); + expect(first).toHaveProperty("sourceSequence", 1); + const refs = [ + first?.actorId, + first?.accountRef, + first?.conversationRef, + first?.messageRef, + first?.targetRef, + ]; + expect(refs).toEqual(refs.map(() => expect.stringMatching(AUDIT_REF_RE))); + expect(new Set(refs).size).toBe(refs.length); + expect(JSON.stringify(first)).not.toContain(rawIdentity); + + const { db } = openOpenClawStateDatabase(database); + const stored = db + .prepare( + `SELECT actor_id, account_ref, conversation_ref, message_ref, target_ref, + session_key, session_id + FROM audit_events`, + ) + .get() as Record; + expect(JSON.stringify(stored)).not.toContain(rawIdentity); + expect(stored.session_key).toBeNull(); + expect(stored.session_id).toBeNull(); + + const keyRow = db.prepare("SELECT key_id, key FROM audit_identity_keys WHERE id = 1").get() as { + key_id: string; + key: Uint8Array; + }; + expect(keyRow.key_id).toMatch(/^[a-f0-9]{32}$/u); + expect(keyRow.key.byteLength).toBe(32); + + closeOpenClawStateDatabaseForTest(); + const second = recordAuditEvent( + messageInput({ + sourceId: "message-source-2", + actorId: rawIdentity, + accountId: rawIdentity, + conversationId: rawIdentity, + messageId: rawIdentity, + targetId: rawIdentity, + }), + database, + ); + expect(second).toMatchObject({ + actorId: first?.actorId, + accountRef: first?.accountRef, + conversationRef: first?.conversationRef, + messageRef: first?.messageRef, + targetRef: first?.targetRef, + }); + }); + + it("fails closed instead of replacing corrupt identity key material", () => { + const database = createDatabaseOptions(); + const { db } = openOpenClawStateDatabase(database); + db.prepare( + "INSERT INTO audit_identity_keys (id, key_id, key, created_at) VALUES (?, ?, ?, ?)", + ).run(1, "a".repeat(32), Buffer.alloc(31), Date.now()); + + expect(() => recordAuditEvent(messageInput(), database)).toThrow( + "audit identity key is corrupt", + ); + const count = db.prepare("SELECT COUNT(*) AS count FROM audit_events").get() as { + count: number; + }; + expect(count.count).toBe(0); + expect( + ( + db.prepare("SELECT length(key) AS length FROM audit_identity_keys WHERE id = 1").get() as { + length: number; + } + ).length, + ).toBe(31); + }); + + it("fails closed when retained message references lose their identity key", () => { + const database = createDatabaseOptions(); + const first = recordAuditEvent(messageInput(), database); + expect(first?.messageRef).toMatch(AUDIT_REF_RE); + + closeOpenClawStateDatabaseForTest(); + const { db } = openOpenClawStateDatabase(database); + db.prepare("DELETE FROM audit_identity_keys WHERE id = 1").run(); + closeOpenClawStateDatabaseForTest(); + + expect(() => + recordAuditEvent(messageInput({ sourceId: "message-after-key-loss" }), database), + ).toThrow("audit identity key is missing"); + const reopened = openOpenClawStateDatabase(database).db; + expect( + ( + reopened.prepare("SELECT COUNT(*) AS count FROM audit_identity_keys").get() as { + count: number; + } + ).count, + ).toBe(0); + }); + + it("forgets a transaction-local identity key after a rolled-back event insert", () => { + const database = createDatabaseOptions(); + const { db } = openOpenClawStateDatabase(database); + db.exec(` + CREATE TABLE audit_rollback_parent (id INTEGER PRIMARY KEY); + CREATE TABLE audit_rollback_child ( + parent_id INTEGER NOT NULL, + FOREIGN KEY (parent_id) REFERENCES audit_rollback_parent(id) + DEFERRABLE INITIALLY DEFERRED + ); + CREATE TRIGGER reject_audit_event + AFTER INSERT ON audit_events + BEGIN + INSERT INTO audit_rollback_child (parent_id) VALUES (1); + END; + `); + + expect(() => recordAuditEvent(messageInput(), database)).toThrow(/FOREIGN KEY/u); + expect( + (db.prepare("SELECT COUNT(*) AS count FROM audit_identity_keys").get() as { count: number }) + .count, + ).toBe(0); + db.exec(` + DROP TRIGGER reject_audit_event; + DROP TABLE audit_rollback_child; + DROP TABLE audit_rollback_parent; + `); + + const committed = recordAuditEvent( + messageInput({ sourceId: "message-after-rollback", messageId: "stable-message" }), + database, + ); + expect(committed?.messageRef).toMatch(AUDIT_REF_RE); + expect( + (db.prepare("SELECT COUNT(*) AS count FROM audit_identity_keys").get() as { count: number }) + .count, + ).toBe(1); + + closeOpenClawStateDatabaseForTest(); + const reopened = recordAuditEvent( + messageInput({ sourceId: "message-after-reopen", messageId: "stable-message" }), + database, + ); + expect(reopened?.messageRef).toBe(committed?.messageRef); + }); + + it("scopes platform message references to their conversation", () => { + const database = createDatabaseOptions(); + const first = recordAuditEvent( + messageInput({ + sourceId: "message-conversation-a-1", + conversationId: "conversation-a", + messageId: "42", + }), + database, + ); + const otherConversation = recordAuditEvent( + messageInput({ + sourceId: "message-conversation-b-1", + conversationId: "conversation-b", + messageId: "42", + }), + database, + ); + const sameConversation = recordAuditEvent( + messageInput({ + sourceId: "message-conversation-a-2", + conversationId: "conversation-a", + messageId: "42", + }), + database, + ); + + expect(first?.messageRef).toMatch(AUDIT_REF_RE); + expect(otherConversation?.messageRef).toMatch(AUDIT_REF_RE); + expect(otherConversation?.messageRef).not.toBe(first?.messageRef); + expect(sameConversation?.messageRef).toBe(first?.messageRef); + }); + + it("uses the outbound target as the conversation correlation fallback", () => { + const database = createDatabaseOptions(); + const event = recordAuditEvent( + outboundMessageInput( + { + status: "failed", + outcome: "failed", + errorCode: "message_delivery_failed", + failureStage: "platform_send", + }, + { + sourceId: "message-outbound-target-fallback", + conversationId: undefined, + targetId: "target-conversation", + }, + ), + database, + ); + + expect(event?.conversationRef).toMatch(AUDIT_REF_RE); + expect(event?.targetRef).toMatch(AUDIT_REF_RE); + expect(event?.conversationRef).not.toBe(event?.targetRef); + }); + + it("rejects persisted message terminal combinations outside the closed contract", () => { + const database = createDatabaseOptions(); + recordAuditEvent(messageInput(), database); + const { db } = openOpenClawStateDatabase(database); + db.prepare("UPDATE audit_events SET error_code = ? WHERE kind = 'message'").run( + "message_processing_failed", + ); + + expect(() => + listAuditEvents({ database, limit: 10, filters: { includeMessages: true } }), + ).toThrow("corrupt audit event row 1: unexpected error_code"); + }); + + it("rejects delivery kind on terminals where no payload was proven delivered", () => { + const database = createDatabaseOptions(); + recordAuditEvent( + outboundMessageInput({ + status: "blocked", + outcome: "suppressed", + reasonCode: "no_visible_payload", + }), + database, + ); + const { db } = openOpenClawStateDatabase(database); + db.prepare("UPDATE audit_events SET delivery_kind = 'text' WHERE kind = 'message'").run(); + + expect(() => + listAuditEvents({ database, limit: 10, filters: { includeMessages: true } }), + ).toThrow("corrupt audit event row 1: unexpected delivery_kind"); + }); + + it("rejects a persisted channel-sender actor without a keyed reference", () => { + const database = createDatabaseOptions(); + recordAuditEvent(messageInput(), database); + const { db } = openOpenClawStateDatabase(database); + db.prepare("UPDATE audit_events SET actor_id = ? WHERE kind = 'message'").run("raw-sender"); + + expect(() => + listAuditEvents({ database, limit: 10, filters: { includeMessages: true } }), + ).toThrow("corrupt audit event row 1: invalid actorId"); + }); + + it("keeps message rows opt-in while supporting message filters", () => { + const database = createDatabaseOptions(); + const now = Date.now(); + recordAuditEvent(runInput({ occurredAt: now }), database); + recordAuditEvent(messageInput({ occurredAt: now + 1 }), database); + recordAuditEvent( + outboundMessageInput( + { status: "succeeded", outcome: "sent" }, + { + sourceId: "message-source-2", + occurredAt: now + 2, + channel: "slack", + conversationKind: "channel", + }, + ), + database, + ); + + expect(listAuditEvents({ database, limit: 10 }).events.map((event) => event.kind)).toEqual([ + "agent_run", + ]); + expect( + listAuditEvents({ database, limit: 10, filters: { includeMessages: true } }).events.map( + (event) => event.kind, + ), + ).toEqual(["message", "message", "agent_run"]); + expect( + listAuditEvents({ + database, + limit: 10, + filters: { + kind: "message", + includeMessages: false, + direction: "inbound", + channel: "telegram", + }, + }).events, + ).toEqual([expect.objectContaining({ direction: "inbound", channel: "telegram" })]); + }); +}); diff --git a/src/audit/audit-event-store.ts b/src/audit/audit-event-store.ts new file mode 100644 index 0000000000000000000000000000000000000000..058c8e7dcdba8bc74814d2a2942e0bf758327bc7 --- /dev/null +++ b/src/audit/audit-event-store.ts @@ -0,0 +1,741 @@ +/** SQLite persistence and stable cursor queries for metadata-only audit events. */ +import { randomUUID } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { AUDIT_ACTIVITY_MESSAGE_KIND } from "../../packages/gateway-protocol/src/schema/audit-activity.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { normalizeSqliteNumber } from "../infra/sqlite-number.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { + AUDIT_EVENT_SCHEMA_VERSION, + AUDIT_INBOUND_MESSAGE_COMPLETED_REASONS, + AUDIT_INBOUND_MESSAGE_SKIPPED_REASONS, + AUDIT_OUTBOUND_MESSAGE_SUPPRESSED_REASONS, + isOutboundMessageProgressInput, + type AgentRunAuditEventRecord, + type AuditEventInput, + type AuditEventListFilters, + type AuditEventListPage, + type AuditEventRecord, + type InboundMessageAuditEventRecord, + type MessageAuditEventInput, + type OutboundMessageAuditEventRecord, + type ToolActionAuditEventRecord, +} from "./audit-event-types.js"; +import { + clearAuditIdentityKeyCacheForDatabase, + loadOrCreateAuditIdentityKey, + pseudonymizeAuditIdentity, +} from "./audit-identity.js"; +import { + ensureTerminalMessageExecutionBindingSchema, + planMessageExecutionBinding, + recordConfirmedTerminalMessageExecutionBinding, +} from "./message-execution-binding.js"; + +type AuditEventsTable = OpenClawStateKyselyDatabase["audit_events"]; +type AuditDatabase = Pick; +type AuditEventRow = Selectable; + +export const AUDIT_EVENT_RETENTION_MS = 30 * 24 * 60 * 60_000; +const AUDIT_EVENT_MAX_ROWS = 100_000; +const AUDIT_EVENT_PRUNE_BATCH_ROWS = 1_024; +// The single audit writer owns one DB handle. Invalidate on out-of-band +// maintenance or rollback so the hot path avoids a 100k-row scan per message. +const auditEventRowCounts = new WeakMap(); + +function getAuditKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +const RUN_ACTIONS = ["agent.run.started", "agent.run.finished"] as const; +const TOOL_ACTIONS = ["tool.action.started", "tool.action.finished"] as const; +const CONVERSATION_KINDS = ["direct", "group", "channel", "unknown"] as const; +const DELIVERY_KINDS = ["text", "media", "other"] as const; +const FAILURE_STAGES = ["platform_send", "queue", "unknown"] as const; +const AUDIT_HMAC_REF_RE = /^hmac-sha256:v1:[a-f0-9]{32}:[a-f0-9]{64}$/u; + +const MESSAGE_COLUMNS = [ + "direction", + "channel", + "conversation_kind", + "message_outcome", + "reason_code", + "delivery_kind", + "failure_stage", + "duration_ms", + "result_count", + "account_ref", + "conversation_ref", + "message_ref", + "target_ref", +] as const satisfies readonly (keyof AuditEventRow)[]; + +function corruptAuditRow(row: AuditEventRow, problem: string): never { + const sequence = normalizeSqliteNumber(row.sequence); + const location = sequence === undefined ? "" : ` ${sequence}`; + throw new Error(`corrupt audit event row${location}: ${problem}`); +} + +function requiredInteger( + row: AuditEventRow, + value: number | bigint | null, + field: string, + minimum: number, +): number { + const normalized = normalizeSqliteNumber(value); + if (normalized === undefined || !Number.isSafeInteger(normalized) || normalized < minimum) { + corruptAuditRow(row, `invalid ${field}`); + } + return normalized; +} + +function optionalInteger( + row: AuditEventRow, + value: number | bigint | null, + field: string, + minimum: number, +): number | undefined { + if (value === null) { + return undefined; + } + return requiredInteger(row, value, field, minimum); +} + +function requiredText(row: AuditEventRow, value: unknown, field: string): string { + if (typeof value !== "string" || value.length === 0) { + corruptAuditRow(row, `invalid ${field}`); + } + return value; +} + +function optionalText(row: AuditEventRow, value: unknown, field: string): string | undefined { + if (value === null || value === undefined) { + return undefined; + } + return requiredText(row, value, field); +} + +function requiredEnum( + row: AuditEventRow, + value: unknown, + field: string, + allowed: readonly Value[], +): Value { + for (const candidate of allowed) { + if (value === candidate) { + return candidate; + } + } + return corruptAuditRow(row, `invalid ${field}`); +} + +function optionalEnum( + row: AuditEventRow, + value: unknown, + field: string, + allowed: readonly Value[], +): Value | undefined { + if (value === null || value === undefined) { + return undefined; + } + return requiredEnum(row, value, field, allowed); +} + +function requiredHmacRef(row: AuditEventRow, value: unknown, field: string): string { + const ref = requiredText(row, value, field); + if (!AUDIT_HMAC_REF_RE.test(ref)) { + corruptAuditRow(row, `invalid ${field}`); + } + return ref; +} + +function optionalHmacRef(row: AuditEventRow, value: unknown, field: string): string | undefined { + if (value === null || value === undefined) { + return undefined; + } + return requiredHmacRef(row, value, field); +} + +function requireNull(row: AuditEventRow, field: keyof AuditEventRow): void { + if (row[field] !== null) { + corruptAuditRow(row, `unexpected ${field}`); + } +} + +function requireNullColumns(row: AuditEventRow, fields: readonly (keyof AuditEventRow)[]): void { + for (const field of fields) { + requireNull(row, field); + } +} + +function parseAuditRecordBase(row: AuditEventRow) { + const schemaVersion = requiredInteger(row, row.schema_version, "schemaVersion", 1); + if (schemaVersion !== AUDIT_EVENT_SCHEMA_VERSION) { + corruptAuditRow(row, `unsupported schemaVersion ${schemaVersion}`); + } + return { + schemaVersion, + sequence: requiredInteger(row, row.sequence, "sequence", 1), + eventId: requiredText(row, row.event_id, "eventId"), + sourceSequence: requiredInteger(row, row.source_sequence, "sourceSequence", 1), + occurredAt: requiredInteger(row, row.occurred_at, "occurredAt", 0), + redaction: "metadata_only" as const, + }; +} + +function parseAgentRecordFields(row: AuditEventRow) { + requireNullColumns(row, MESSAGE_COLUMNS); + return { + ...parseAuditRecordBase(row), + actorType: requiredEnum(row, row.actor_type, "actorType", ["agent", "system"]), + actorId: requiredText(row, row.actor_id, "actorId"), + agentId: requiredText(row, row.agent_id, "agentId"), + ...(optionalText(row, row.session_key, "sessionKey") !== undefined + ? { sessionKey: requiredText(row, row.session_key, "sessionKey") } + : {}), + ...(optionalText(row, row.session_id, "sessionId") !== undefined + ? { sessionId: requiredText(row, row.session_id, "sessionId") } + : {}), + runId: requiredText(row, row.run_id, "runId"), + }; +} + +function parseAgentRunRow(row: AuditEventRow): AgentRunAuditEventRecord { + requireNull(row, "tool_call_id"); + requireNull(row, "tool_name"); + const common = { ...parseAgentRecordFields(row), kind: "agent_run" as const }; + const action = requiredEnum(row, row.action, "action", RUN_ACTIONS); + if (action === "agent.run.started") { + requiredEnum(row, row.status, "status", ["started"]); + requireNull(row, "error_code"); + return { ...common, action, status: "started" }; + } + if (row.status === "succeeded") { + requireNull(row, "error_code"); + return { ...common, action, status: "succeeded" }; + } + const terminal = + row.status === "failed" + ? { status: "failed" as const, errorCode: "run_failed" as const } + : row.status === "cancelled" + ? { status: "cancelled" as const, errorCode: "run_cancelled" as const } + : row.status === "timed_out" + ? { status: "timed_out" as const, errorCode: "run_timed_out" as const } + : row.status === "blocked" + ? { status: "blocked" as const, errorCode: "run_blocked" as const } + : corruptAuditRow(row, "invalid run terminal status"); + requiredEnum(row, row.error_code, "errorCode", [terminal.errorCode]); + return { ...common, action, ...terminal }; +} + +function parseToolActionRow(row: AuditEventRow): ToolActionAuditEventRecord { + const toolCallId = optionalText(row, row.tool_call_id, "toolCallId"); + const toolName = optionalText(row, row.tool_name, "toolName"); + const common = { + ...parseAgentRecordFields(row), + kind: "tool_action" as const, + ...(toolCallId ? { toolCallId } : {}), + ...(toolName ? { toolName } : {}), + }; + const action = requiredEnum(row, row.action, "action", TOOL_ACTIONS); + if (action === "tool.action.started") { + requiredEnum(row, row.status, "status", ["started"]); + requireNull(row, "error_code"); + return { ...common, action, status: "started" }; + } + if (row.status === "succeeded") { + requireNull(row, "error_code"); + return { ...common, action, status: "succeeded" }; + } + const terminal = + row.status === "failed" + ? { status: "failed" as const, errorCode: "tool_failed" as const } + : row.status === "cancelled" + ? { status: "cancelled" as const, errorCode: "tool_cancelled" as const } + : row.status === "timed_out" + ? { status: "timed_out" as const, errorCode: "tool_timed_out" as const } + : row.status === "blocked" + ? { status: "blocked" as const, errorCode: "tool_blocked" as const } + : row.status === "unknown" + ? { status: "unknown" as const, errorCode: "tool_outcome_unknown" as const } + : corruptAuditRow(row, "invalid tool terminal status"); + requiredEnum(row, row.error_code, "errorCode", [terminal.errorCode]); + return { ...common, action, ...terminal }; +} + +function parseMessageRecordFields(row: AuditEventRow) { + requireNullColumns(row, ["session_key", "session_id", "tool_call_id", "tool_name"]); + const agentId = optionalText(row, row.agent_id, "agentId"); + const runId = optionalText(row, row.run_id, "runId"); + const durationMs = optionalInteger(row, row.duration_ms, "durationMs", 0); + const resultCount = optionalInteger(row, row.result_count, "resultCount", 0); + const accountRef = optionalHmacRef(row, row.account_ref, "accountRef"); + const conversationRef = optionalHmacRef(row, row.conversation_ref, "conversationRef"); + const messageRef = optionalHmacRef(row, row.message_ref, "messageRef"); + const targetRef = optionalHmacRef(row, row.target_ref, "targetRef"); + return { + ...parseAuditRecordBase(row), + kind: "message" as const, + channel: requiredText(row, row.channel, "channel"), + conversationKind: requiredEnum( + row, + row.conversation_kind, + "conversationKind", + CONVERSATION_KINDS, + ), + ...(agentId ? { agentId } : {}), + ...(runId ? { runId } : {}), + ...(durationMs !== undefined ? { durationMs } : {}), + ...(resultCount !== undefined ? { resultCount } : {}), + ...(accountRef ? { accountRef } : {}), + ...(conversationRef ? { conversationRef } : {}), + ...(messageRef ? { messageRef } : {}), + ...(targetRef ? { targetRef } : {}), + }; +} + +function parseInboundMessageRow(row: AuditEventRow): InboundMessageAuditEventRecord { + requiredEnum(row, row.action, "action", ["message.inbound.processed"]); + requiredEnum(row, row.direction, "direction", ["inbound"]); + requireNull(row, "delivery_kind"); + requireNull(row, "failure_stage"); + const actorType = requiredEnum(row, row.actor_type, "actorType", ["channel_sender", "system"]); + const actorId = + actorType === "channel_sender" + ? requiredHmacRef(row, row.actor_id, "actorId") + : requiredText(row, row.actor_id, "actorId"); + const common = { + ...parseMessageRecordFields(row), + action: "message.inbound.processed" as const, + direction: "inbound" as const, + actorType, + actorId, + }; + if (row.status === "succeeded") { + requiredEnum(row, row.message_outcome, "outcome", ["completed"]); + requireNull(row, "error_code"); + const reasonCode = optionalEnum( + row, + row.reason_code, + "reasonCode", + AUDIT_INBOUND_MESSAGE_COMPLETED_REASONS, + ); + return { + ...common, + status: "succeeded", + outcome: "completed", + ...(reasonCode ? { reasonCode } : {}), + }; + } + if (row.status === "blocked") { + requiredEnum(row, row.message_outcome, "outcome", ["skipped"]); + requireNull(row, "error_code"); + const reasonCode = optionalEnum( + row, + row.reason_code, + "reasonCode", + AUDIT_INBOUND_MESSAGE_SKIPPED_REASONS, + ); + return { + ...common, + status: "blocked", + outcome: "skipped", + ...(reasonCode ? { reasonCode } : {}), + }; + } + if (row.status === "failed") { + requiredEnum(row, row.message_outcome, "outcome", ["failed"]); + requiredEnum(row, row.error_code, "errorCode", ["message_processing_failed"]); + const reasonCode = optionalEnum(row, row.reason_code, "reasonCode", [ + "acp_dispatch_failed", + "plugin_bound_error", + ]); + return { + ...common, + status: "failed", + outcome: "failed", + errorCode: "message_processing_failed", + ...(reasonCode ? { reasonCode } : {}), + }; + } + return corruptAuditRow(row, "invalid inbound status"); +} + +function parseOutboundMessageRow(row: AuditEventRow): OutboundMessageAuditEventRecord { + const action = requiredEnum(row, row.action, "action", [ + "message.outbound.queued", + "message.outbound.platform-started", + "message.outbound.finished", + ]); + requiredEnum(row, row.direction, "direction", ["outbound"]); + const actorType = requiredEnum(row, row.actor_type, "actorType", ["agent", "system"]); + const actorId = requiredText(row, row.actor_id, "actorId"); + const commonFields = parseMessageRecordFields(row); + const common = { + ...commonFields, + action, + direction: "outbound" as const, + actorType, + actorId, + }; + if (row.status === "started") { + requireNull(row, "delivery_kind"); + requireNullColumns(row, ["error_code", "reason_code", "failure_stage"]); + if (action === "message.outbound.queued") { + requiredEnum(row, row.message_outcome, "outcome", ["queued"]); + return { ...common, action, status: "started", outcome: "queued" }; + } + if (action === "message.outbound.platform-started") { + requiredEnum(row, row.message_outcome, "outcome", ["platform_started"]); + return { ...common, action, status: "started", outcome: "platform_started" }; + } + return corruptAuditRow(row, "invalid outbound lifecycle action"); + } + requiredEnum(row, action, "action", ["message.outbound.finished"]); + const terminalCommon = { ...common, action: "message.outbound.finished" as const }; + if (row.status === "succeeded") { + const deliveryKind = optionalEnum(row, row.delivery_kind, "deliveryKind", DELIVERY_KINDS); + requiredEnum(row, row.message_outcome, "outcome", ["sent"]); + requireNullColumns(row, ["error_code", "reason_code", "failure_stage"]); + return { + ...terminalCommon, + status: "succeeded", + outcome: "sent", + ...(deliveryKind ? { deliveryKind } : {}), + }; + } + if (row.status === "blocked") { + requireNull(row, "delivery_kind"); + requiredEnum(row, row.message_outcome, "outcome", ["suppressed"]); + requireNullColumns(row, ["error_code", "failure_stage"]); + const reasonCode = requiredEnum( + row, + row.reason_code, + "reasonCode", + AUDIT_OUTBOUND_MESSAGE_SUPPRESSED_REASONS, + ); + return { + ...terminalCommon, + status: "blocked", + outcome: "suppressed", + reasonCode, + }; + } + if (row.status === "failed") { + const deliveryKind = optionalEnum(row, row.delivery_kind, "deliveryKind", DELIVERY_KINDS); + requiredEnum(row, row.message_outcome, "outcome", ["failed"]); + requireNull(row, "reason_code"); + const errorCode = requiredEnum(row, row.error_code, "errorCode", [ + "message_delivery_failed", + "message_delivery_partial_failure", + ]); + const failureStage = requiredEnum(row, row.failure_stage, "failureStage", FAILURE_STAGES); + return { + ...terminalCommon, + status: "failed", + outcome: "failed", + errorCode, + failureStage, + ...(deliveryKind ? { deliveryKind } : {}), + }; + } + if (row.status === "unknown") { + requireNull(row, "delivery_kind"); + requiredEnum(row, row.message_outcome, "outcome", ["unknown"]); + requireNullColumns(row, ["error_code", "reason_code"]); + const failureStage = requiredEnum(row, row.failure_stage, "failureStage", FAILURE_STAGES); + return { + ...terminalCommon, + status: "unknown", + outcome: "unknown", + failureStage, + }; + } + return corruptAuditRow(row, "invalid outbound status"); +} + +export function rowToAuditEvent(row: AuditEventRow): AuditEventRecord { + if (row.kind === "agent_run") { + return parseAgentRunRow(row); + } + if (row.kind === "tool_action") { + return parseToolActionRow(row); + } + if (row.kind !== "message") { + corruptAuditRow(row, "invalid kind"); + } + if (row.direction === "inbound") { + return parseInboundMessageRow(row); + } + if (row.direction === "outbound") { + return parseOutboundMessageRow(row); + } + return corruptAuditRow(row, "invalid message direction"); +} + +function projectMessageIdentities(db: DatabaseSync, input: MessageAuditEventInput) { + const identity = loadOrCreateAuditIdentityKey(db); + const conversationId = + input.conversationId ?? (input.direction === "outbound" ? input.targetId : undefined); + const ref = ( + kind: Parameters[0]["kind"], + value: string | undefined, + ) => + pseudonymizeAuditIdentity({ + identity, + kind, + channel: input.channel, + ...(kind !== "account" && input.accountId !== undefined + ? { accountId: input.accountId } + : {}), + ...(kind === "message" && conversationId !== undefined ? { conversationId } : {}), + value, + }); + return { + actorId: input.actorType === "channel_sender" ? ref("actor", input.actorId) : input.actorId, + accountRef: ref("account", input.accountId), + conversationRef: ref("conversation", conversationId), + messageRef: ref("message", input.messageId), + targetRef: ref("target", input.targetId), + }; +} + +function bindAuditEvent(db: DatabaseSync, input: AuditEventInput): Insertable { + const message = + input.kind === AUDIT_ACTIVITY_MESSAGE_KIND ? projectMessageIdentities(db, input) : undefined; + return { + event_id: randomUUID(), + source_id: input.sourceId, + source_sequence: input.sourceSequence, + schema_version: AUDIT_EVENT_SCHEMA_VERSION, + occurred_at: input.occurredAt, + kind: input.kind, + action: input.action, + status: input.status, + error_code: input.errorCode ?? null, + actor_type: input.actorType, + actor_id: message?.actorId ?? input.actorId, + agent_id: input.agentId ?? null, + session_key: input.kind === AUDIT_ACTIVITY_MESSAGE_KIND ? null : (input.sessionKey ?? null), + session_id: input.kind === AUDIT_ACTIVITY_MESSAGE_KIND ? null : (input.sessionId ?? null), + run_id: input.runId ?? null, + tool_call_id: input.kind === "tool_action" ? (input.toolCallId ?? null) : null, + tool_name: input.kind === "tool_action" ? input.toolName : null, + direction: input.kind === AUDIT_ACTIVITY_MESSAGE_KIND ? input.direction : null, + channel: input.kind === AUDIT_ACTIVITY_MESSAGE_KIND ? input.channel : null, + conversation_kind: input.kind === "message" ? input.conversationKind : null, + message_outcome: input.kind === "message" ? input.outcome : null, + reason_code: input.kind === "message" ? (input.reasonCode ?? null) : null, + delivery_kind: input.kind === "message" ? (input.deliveryKind ?? null) : null, + failure_stage: input.kind === "message" ? (input.failureStage ?? null) : null, + duration_ms: input.kind === "message" ? (input.durationMs ?? null) : null, + result_count: input.kind === "message" ? (input.resultCount ?? null) : null, + account_ref: message?.accountRef ?? null, + conversation_ref: message?.conversationRef ?? null, + message_ref: message?.messageRef ?? null, + target_ref: message?.targetRef ?? null, + }; +} + +function countAuditEvents(db: DatabaseSync): number { + const row = executeSqliteQueryTakeFirstSync( + db, + getAuditKysely(db) + .selectFrom("audit_events") + .select((expression) => expression.fn.countAll().as("count")), + ); + return normalizeSqliteNumber(row?.count ?? null) ?? 0; +} + +function deleteExpiredAuditEvents(db: DatabaseSync, now: number) { + const kysely = getAuditKysely(db); + const expiredSequences = kysely + .selectFrom("audit_events") + .select("sequence") + .where("occurred_at", "<", now - AUDIT_EVENT_RETENTION_MS) + .orderBy("occurred_at", "asc") + .orderBy("sequence", "asc") + .limit(AUDIT_EVENT_PRUNE_BATCH_ROWS); + return executeSqliteQuerySync( + db, + kysely.deleteFrom("audit_events").where("sequence", "in", expiredSequences), + ); +} + +function pruneAuditEventsAfterInsert(db: DatabaseSync, now: number): void { + const kysely = getAuditKysely(db); + const expired = deleteExpiredAuditEvents(db, now); + const cachedCount = auditEventRowCounts.get(db); + let rowCount = + cachedCount === undefined + ? countAuditEvents(db) + : Math.max(0, cachedCount + 1 - Number(expired.numAffectedRows ?? 0n)); + if (rowCount <= AUDIT_EVENT_MAX_ROWS) { + auditEventRowCounts.set(db, rowCount); + return; + } + const retainedRows = Math.max(0, AUDIT_EVENT_MAX_ROWS - AUDIT_EVENT_PRUNE_BATCH_ROWS); + const overflowRow = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("audit_events") + .select("sequence") + .orderBy("sequence", "desc") + .offset(retainedRows) + .limit(1), + ); + const sequenceCutoff = overflowRow ? normalizeSqliteNumber(overflowRow.sequence) : undefined; + if (sequenceCutoff !== undefined) { + const pruned = executeSqliteQuerySync( + db, + kysely.deleteFrom("audit_events").where("sequence", "<=", sequenceCutoff), + ); + rowCount = Math.max(0, rowCount - Number(pruned.numAffectedRows ?? 0n)); + } + auditEventRowCounts.set(db, rowCount); +} + +/** Persist one projected event idempotently and prune fixed retention bounds. */ +export function recordAuditEvent( + input: AuditEventInput, + options: OpenClawStateDatabaseOptions = {}, +): AuditEventRecord | undefined { + if (isOutboundMessageProgressInput(input)) { + throw new Error("outbound message progress belongs to its companion store"); + } + const executionToken = + input.kind === "message" && input.direction === "outbound" + ? planMessageExecutionBinding(input.executionIdentityToken, input.runId) + : undefined; + if (executionToken) { + ensureTerminalMessageExecutionBindingSchema(options); + } + let countCacheDatabase: DatabaseSync | undefined; + try { + return runOpenClawStateWriteTransaction(({ db }) => { + countCacheDatabase = db; + // Read losslessly so Node's rowid decoding cannot preempt the safe-integer guard. + const insert = executeSqliteQueryTakeFirstSync( + db, + getAuditKysely(db) + .insertInto("audit_events") + .values(bindAuditEvent(db, input)) + .onConflict((conflict) => conflict.column("source_id").doNothing()) + .returning((eb) => eb.cast("sequence", "text").as("sequence")), + ); + if (insert === undefined) { + return undefined; + } + const insertedSequence = Number(insert.sequence); + if (!Number.isSafeInteger(insertedSequence) || insertedSequence < 1) { + throw new Error("audit event sequence is outside the supported integer range"); + } + pruneAuditEventsAfterInsert(db, Date.now()); + const row = executeSqliteQueryTakeFirstSync( + db, + getAuditKysely(db) + .selectFrom("audit_events") + .selectAll() + .where("sequence", "=", insertedSequence), + ); + recordConfirmedTerminalMessageExecutionBinding(db, { + eventId: row?.event_id, + token: executionToken, + }); + return row ? rowToAuditEvent(row) : undefined; + }, options); + } catch (error) { + if (countCacheDatabase) { + auditEventRowCounts.delete(countCacheDatabase); + clearAuditIdentityKeyCacheForDatabase(countCacheDatabase); + } + throw error; + } +} + +/** List newest-first records using a stable sequence cursor. */ +export function listAuditEvents(params: { + filters?: AuditEventListFilters; + cursor?: number; + limit: number; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): AuditEventListPage { + const { db } = openOpenClawStateDatabase(params.database); + const filters = params.filters ?? {}; + const retainedAfter = (params.now ?? Date.now()) - AUDIT_EVENT_RETENTION_MS; + let query = getAuditKysely(db) + .selectFrom("audit_events") + .selectAll() + .where("occurred_at", ">=", retainedAfter) + // Nonterminal outbound facts belong to the lazy progress owner. Excluding + // transitional rows keeps the released activity contract terminal-only. + .where("action", "not in", ["message.outbound.queued", "message.outbound.platform-started"]); + if (params.cursor !== undefined) { + query = query.where("sequence", "<", params.cursor); + } + if (filters.agentId) { + query = query.where("agent_id", "=", filters.agentId); + } + if (filters.sessionKey) { + query = query.where("session_key", "=", filters.sessionKey); + } + if (filters.runId) { + query = query.where("run_id", "=", filters.runId); + } + if (filters.kind) { + query = query.where("kind", "=", filters.kind); + } else if (filters.includeMessages !== true) { + query = query.where("kind", "!=", "message"); + } + if (filters.status) { + query = query.where("status", "=", filters.status); + } + if (filters.direction) { + query = query.where("direction", "=", filters.direction); + } + if (filters.channel) { + query = query.where("channel", "=", filters.channel); + } + if (filters.after !== undefined) { + query = query.where("occurred_at", ">=", filters.after); + } + if (filters.before !== undefined) { + query = query.where("occurred_at", "<=", filters.before); + } + const rows = executeSqliteQuerySync( + db, + query.orderBy("sequence", "desc").limit(params.limit + 1), + ).rows; + const hasMore = rows.length > params.limit; + const pageRows = hasMore ? rows.slice(0, params.limit) : rows; + const events = pageRows.map(rowToAuditEvent); + return { + events, + ...(hasMore && events.length > 0 ? { nextCursor: events[events.length - 1]?.sequence } : {}), + }; +} + +/** Delete one bounded batch during Gateway startup and periodic audit maintenance. */ +export function pruneExpiredAuditEvents( + params: { + now?: number; + database?: OpenClawStateDatabaseOptions; + } = {}, +): number { + return runOpenClawStateWriteTransaction(({ db }) => { + const deleted = deleteExpiredAuditEvents(db, params.now ?? Date.now()); + auditEventRowCounts.delete(db); + return Number(deleted.numAffectedRows ?? 0n); + }, params.database); +} diff --git a/src/audit/audit-event-types.ts b/src/audit/audit-event-types.ts new file mode 100644 index 0000000000000000000000000000000000000000..c74880d5f18c23d445e3c8ebf87ab35581efda47 --- /dev/null +++ b/src/audit/audit-event-types.ts @@ -0,0 +1,357 @@ +/** Versioned metadata-only durable audit contract. */ +import type { ExecutionIdentityAdmissionToken } from "./execution-identity-admission.js"; + +export const AUDIT_EVENT_SCHEMA_VERSION = 1 as const; + +type AuditEventKind = "agent_run" | "tool_action" | "message"; + +type AuditEventStatus = + | "started" + | "succeeded" + | "failed" + | "cancelled" + | "timed_out" + | "blocked" + | "unknown"; + +type AuditMessageDirection = "inbound" | "outbound"; +export type AuditMessageConversationKind = "direct" | "group" | "channel" | "unknown"; +export type AuditMessageDeliveryKind = "text" | "media" | "other"; +export type AuditMessageFailureStage = "platform_send" | "queue" | "unknown"; + +export const AUDIT_INBOUND_MESSAGE_COMPLETED_REASONS = [ + "fast_abort", + "plugin_bound_handled", + "plugin_bound_unavailable", + "plugin_bound_declined", + "before_dispatch_handled", + "acp_dispatch_completed", + "acp_dispatch_empty", + "active_run_injected", +] as const; + +export type AuditInboundMessageCompletedReasonCode = + (typeof AUDIT_INBOUND_MESSAGE_COMPLETED_REASONS)[number]; + +export const AUDIT_INBOUND_MESSAGE_SKIPPED_REASONS = [ + "duplicate", + "reply_operation_active", + "reply_operation_aborted", + "acp_dispatch_aborted", +] as const; + +export type AuditInboundMessageSkippedReasonCode = + (typeof AUDIT_INBOUND_MESSAGE_SKIPPED_REASONS)[number]; + +type AuditInboundMessageFailureReasonCode = "acp_dispatch_failed" | "plugin_bound_error"; + +export const AUDIT_OUTBOUND_MESSAGE_SUPPRESSED_REASONS = [ + "cancelled_by_message_sending_hook", + "cancelled_by_reply_payload_sending_hook", + "empty_after_message_sending_hook", + "empty_after_reply_payload_sending_hook", + "no_visible_payload", +] as const; + +export type AuditOutboundMessageSuppressedReasonCode = + (typeof AUDIT_OUTBOUND_MESSAGE_SUPPRESSED_REASONS)[number]; + +type AuditEventInputBase = { + /** Stable trusted-source identity used only for local replay deduplication. */ + sourceId: string; + sourceSequence: number; + occurredAt: number; +}; + +type AgentAuditAttribution = { + actorType: "agent" | "system"; + actorId: string; + agentId: string; + sessionKey?: string; + sessionId?: string; + runId: string; +}; + +export type AgentRunFinishedAuditTerminal = + | { status: "succeeded"; errorCode?: never } + | { status: "failed"; errorCode: "run_failed" } + | { status: "cancelled"; errorCode: "run_cancelled" } + | { status: "timed_out"; errorCode: "run_timed_out" } + | { status: "blocked"; errorCode: "run_blocked" }; + +type AgentRunAuditLifecycle = + | { action: "agent.run.started"; status: "started"; errorCode?: never } + | ({ action: "agent.run.finished" } & AgentRunFinishedAuditTerminal); + +type AgentRunAuditEventInput = AuditEventInputBase & + AgentAuditAttribution & + AgentRunAuditLifecycle & { kind: "agent_run" }; + +type ToolActionFinishedAuditTerminal = + | { status: "succeeded"; errorCode?: never } + | { status: "failed"; errorCode: "tool_failed" } + | { status: "cancelled"; errorCode: "tool_cancelled" } + | { status: "timed_out"; errorCode: "tool_timed_out" } + | { status: "blocked"; errorCode: "tool_blocked" } + | { status: "unknown"; errorCode: "tool_outcome_unknown" }; + +type ToolActionAuditLifecycle = + | { action: "tool.action.started"; status: "started"; errorCode?: never } + | ({ action: "tool.action.finished" } & ToolActionFinishedAuditTerminal); + +export type ToolActionAuditEventInput = AuditEventInputBase & + AgentAuditAttribution & + ToolActionAuditLifecycle & { + kind: "tool_action"; + toolCallId?: string; + toolName: string; + }; + +type MessageAuditEventInputBase = { + sourceId: string; + sourceSequence: number; + occurredAt: number; + kind: "message"; + actorId: string; + agentId?: string; + runId?: string; + channel: string; + conversationKind: AuditMessageConversationKind; + durationMs?: number; + resultCount?: number; + accountId?: string; + conversationId?: string; + messageId?: string; + targetId?: string; +}; + +type InboundMessageAuditAttribution = { + actorType: "channel_sender" | "system"; + actorId: string; +}; + +type OutboundMessageAuditAttribution = { + actorType: "agent" | "system"; + actorId: string; +}; + +export type InboundMessageAuditTerminal = + | { + status: "succeeded"; + outcome: "completed"; + reasonCode?: AuditInboundMessageCompletedReasonCode; + errorCode?: never; + } + | { + status: "blocked"; + outcome: "skipped"; + reasonCode?: AuditInboundMessageSkippedReasonCode; + errorCode?: never; + } + | { + status: "failed"; + outcome: "failed"; + errorCode: "message_processing_failed"; + reasonCode?: AuditInboundMessageFailureReasonCode; + }; + +type OutboundMessageAuditLifecycle = + | { + action: "message.outbound.queued"; + status: "started"; + outcome: "queued"; + errorCode?: never; + reasonCode?: never; + failureStage?: never; + deliveryKind?: never; + } + | { + action: "message.outbound.platform-started"; + status: "started"; + outcome: "platform_started"; + errorCode?: never; + reasonCode?: never; + failureStage?: never; + deliveryKind?: never; + } + | { + action: "message.outbound.finished"; + status: "succeeded"; + outcome: "sent"; + errorCode?: never; + reasonCode?: never; + failureStage?: never; + deliveryKind?: AuditMessageDeliveryKind; + } + | { + action: "message.outbound.finished"; + status: "blocked"; + outcome: "suppressed"; + reasonCode: AuditOutboundMessageSuppressedReasonCode; + errorCode?: never; + failureStage?: never; + deliveryKind?: never; + } + | { + action: "message.outbound.finished"; + status: "failed"; + outcome: "failed"; + errorCode: "message_delivery_failed" | "message_delivery_partial_failure"; + failureStage: AuditMessageFailureStage; + reasonCode?: never; + deliveryKind?: AuditMessageDeliveryKind; + } + | { + action: "message.outbound.finished"; + status: "unknown"; + outcome: "unknown"; + failureStage: AuditMessageFailureStage; + errorCode?: never; + reasonCode?: never; + deliveryKind?: never; + }; + +export type OutboundMessageProgressInput = MessageAuditEventInputBase & + OutboundMessageAuditAttribution & + Extract< + OutboundMessageAuditLifecycle, + { action: "message.outbound.queued" | "message.outbound.platform-started" } + > & { + direction: "outbound"; + executionIdentityToken?: ExecutionIdentityAdmissionToken; + }; + +/** Raw identifiers exist only on the trusted producer-to-writer boundary. */ +type InboundMessageAuditEventInput = MessageAuditEventInputBase & + InboundMessageAuditAttribution & + InboundMessageAuditTerminal & { + action: "message.inbound.processed"; + direction: "inbound"; + deliveryKind?: never; + failureStage?: never; + }; + +/** Raw identifiers exist only on the trusted producer-to-writer boundary. */ +type OutboundMessageAuditEventInput = MessageAuditEventInputBase & + OutboundMessageAuditAttribution & + OutboundMessageAuditLifecycle & { + direction: "outbound"; + executionIdentityToken?: ExecutionIdentityAdmissionToken; + }; + +export type MessageAuditEventInput = InboundMessageAuditEventInput | OutboundMessageAuditEventInput; + +/** Durable columns accepted from trusted lifecycle projections. */ +export type AuditEventInput = + | AgentRunAuditEventInput + | ToolActionAuditEventInput + | MessageAuditEventInput; + +export function isOutboundMessageProgressInput( + input: AuditEventInput, +): input is OutboundMessageProgressInput { + return ( + input.kind === "message" && + input.direction === "outbound" && + (input.action === "message.outbound.queued" || + input.action === "message.outbound.platform-started") + ); +} + +type AuditEventRecordBase = { + schemaVersion: typeof AUDIT_EVENT_SCHEMA_VERSION; + sequence: number; + eventId: string; + sourceSequence: number; + occurredAt: number; + redaction: "metadata_only"; +}; + +type AgentAuditEventRecordBase = AuditEventRecordBase & { + actorType: "agent" | "system"; + actorId: string; + agentId: string; + sessionKey?: string; + sessionId?: string; + runId: string; + toolCallId?: string; + toolName?: string; + direction?: never; + channel?: never; + conversationKind?: never; + outcome?: never; + reasonCode?: never; + deliveryKind?: never; + failureStage?: never; + durationMs?: never; + resultCount?: never; + accountRef?: never; + conversationRef?: never; + messageRef?: never; + targetRef?: never; +}; + +export type AgentRunAuditEventRecord = AgentAuditEventRecordBase & { + kind: "agent_run"; +} & AgentRunAuditLifecycle; + +export type ToolActionAuditEventRecord = AgentAuditEventRecordBase & { + kind: "tool_action"; +} & ToolActionAuditLifecycle; + +type MessageAuditEventRecordBase = AuditEventRecordBase & { + kind: "message"; + actorId: string; + agentId?: string; + runId?: string; + sessionKey?: never; + sessionId?: never; + toolCallId?: never; + toolName?: never; + channel: string; + conversationKind: AuditMessageConversationKind; + durationMs?: number; + resultCount?: number; + accountRef?: string; + conversationRef?: string; + messageRef?: string; + targetRef?: string; +}; + +export type InboundMessageAuditEventRecord = MessageAuditEventRecordBase & + InboundMessageAuditAttribution & + InboundMessageAuditTerminal & { + action: "message.inbound.processed"; + direction: "inbound"; + }; + +export type OutboundMessageAuditEventRecord = MessageAuditEventRecordBase & + OutboundMessageAuditAttribution & + OutboundMessageAuditLifecycle & { + direction: "outbound"; + }; + +/** Public record returned by the bounded operator read surface. */ +export type AuditEventRecord = + | AgentRunAuditEventRecord + | ToolActionAuditEventRecord + | InboundMessageAuditEventRecord + | OutboundMessageAuditEventRecord; + +export type AuditEventListFilters = { + agentId?: string; + sessionKey?: string; + runId?: string; + kind?: AuditEventKind; + status?: AuditEventStatus; + direction?: AuditMessageDirection; + channel?: string; + includeMessages?: boolean; + after?: number; + before?: number; +}; + +export type AuditEventListPage = { + events: AuditEventRecord[]; + nextCursor?: number; +}; diff --git a/src/audit/audit-recorder.test.ts b/src/audit/audit-recorder.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..078d0ab5acf5d60490cdf1a3a72e91633fc058af --- /dev/null +++ b/src/audit/audit-recorder.test.ts @@ -0,0 +1,89 @@ +import { describe, expect, it } from "vitest"; +import type { AuditEventInput } from "./audit-event-types.js"; +import type { AuditEventWriter } from "./audit-event-writer.js"; +import { createAuditEventRecorder } from "./audit-recorder.js"; +import { emitTrustedMessageAuditEvent } from "./message-audit-events.js"; +import { onTrustedMessageAuditEventForTest as onTrustedMessageAuditEvent } from "./message-audit-events.test-support.js"; + +function captureWriter(inputs: AuditEventInput[]): AuditEventWriter { + return { + ready: Promise.resolve(), + record: (input) => { + inputs.push(input); + return true; + }, + recordExecutionIdentity: () => true, + recordExecutionDecision: () => true, + recordExecutionDecisionWork: () => true, + stop: async () => {}, + }; +} + +function emitMessage(conversationKind: "direct" | "group") { + emitTrustedMessageAuditEvent({ + occurredAt: 1, + kind: "message", + action: "message.inbound.processed", + status: "succeeded", + actorType: "channel_sender", + actorId: "sender-raw", + direction: "inbound", + channel: "telegram", + conversationKind, + outcome: "completed", + }); +} + +describe("message audit recorder", () => { + it("keeps message events off by default policy", async () => { + const inputs: AuditEventInput[] = []; + const recorder = createAuditEventRecorder({ + messageMode: "off", + writer: captureWriter(inputs), + }); + const unsubscribe = onTrustedMessageAuditEvent(recorder.recordMessage); + + emitMessage("direct"); + + unsubscribe(); + await recorder.stop(); + expect(inputs).toEqual([]); + }); + + it("records only known direct conversations in direct mode", async () => { + const inputs: AuditEventInput[] = []; + const recorder = createAuditEventRecorder({ + messageMode: "direct", + writer: captureWriter(inputs), + }); + const unsubscribe = onTrustedMessageAuditEvent(recorder.recordMessage); + + emitMessage("group"); + emitMessage("direct"); + + unsubscribe(); + await recorder.stop(); + expect(inputs).toHaveLength(1); + expect(inputs[0]).toMatchObject({ + kind: "message", + conversationKind: "direct", + sourceId: expect.stringMatching(/^message:/u), + sourceSequence: 1, + }); + }); + + it("records group metadata only in all mode", async () => { + const inputs: AuditEventInput[] = []; + const recorder = createAuditEventRecorder({ + messageMode: "all", + writer: captureWriter(inputs), + }); + const unsubscribe = onTrustedMessageAuditEvent(recorder.recordMessage); + + emitMessage("group"); + + unsubscribe(); + await recorder.stop(); + expect(inputs[0]).toMatchObject({ kind: "message", conversationKind: "group" }); + }); +}); diff --git a/src/audit/audit-recorder.ts b/src/audit/audit-recorder.ts new file mode 100644 index 0000000000000000000000000000000000000000..0a161f68497311f531305c9ea61fcd86c7cd93d2 --- /dev/null +++ b/src/audit/audit-recorder.ts @@ -0,0 +1,69 @@ +/** Gateway-owned recorder joining trusted run, tool, and message lifecycle streams. */ +import { randomUUID } from "node:crypto"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { + createAgentEventAuditRecorder, + type AgentEventAuditRecorder, +} from "./agent-event-audit.js"; +import type { AuditMessageMode } from "./audit-config.js"; +import { createAuditEventWriter, type AuditEventWriter } from "./audit-event-writer.js"; +import type { ExecutionIdentityAdmissionWork } from "./execution-identity-admission.js"; +import type { TrustedMessageAuditEvent } from "./message-audit-events.js"; + +const log = createSubsystemLogger("audit/events"); +let persistenceFailureWarned = false; + +type AuditEventRecorder = AgentEventAuditRecorder & { + recordMessage: (event: TrustedMessageAuditEvent) => void; + recordExecutionIdentity: (work: ExecutionIdentityAdmissionWork) => boolean; + recordExecutionDecision: AuditEventWriter["recordExecutionDecision"]; + recordExecutionDecisionWork: AuditEventWriter["recordExecutionDecisionWork"]; +}; + +export function createAuditEventRecorder(options: { + messageMode: AuditMessageMode; + writer?: AuditEventWriter; + stateDir?: string; + terminalSettleMs?: number; +}): AuditEventRecorder { + let nextAcceptedMessageSequence = 0; + const writer = + options.writer ?? + createAuditEventWriter({ + ...(options.stateDir ? { stateDir: options.stateDir } : {}), + onContention: (message) => log.warn(message), + onError: (error) => { + if (!persistenceFailureWarned) { + persistenceFailureWarned = true; + log.warn(`audit event persistence failed: ${error}`); + } + }, + }); + const agentRecorder = createAgentEventAuditRecorder({ + writer, + ...(options.terminalSettleMs !== undefined + ? { terminalSettleMs: options.terminalSettleMs } + : {}), + }); + + return { + ...agentRecorder, + recordExecutionIdentity: writer.recordExecutionIdentity, + recordExecutionDecision: writer.recordExecutionDecision, + recordExecutionDecisionWork: writer.recordExecutionDecisionWork, + recordMessage: (event) => { + if (options.messageMode === "off") { + return; + } + if (options.messageMode === "direct" && event.conversationKind !== "direct") { + return; + } + nextAcceptedMessageSequence += 1; + writer.record({ + ...event, + sourceId: event.sourceId?.trim() || `message:${randomUUID()}`, + sourceSequence: nextAcceptedMessageSequence, + }); + }, + }; +} diff --git a/src/audit/execution-decision-facts.test.ts b/src/audit/execution-decision-facts.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c8cd40e37a591bbcb8448cc095da41d15c69047c --- /dev/null +++ b/src/audit/execution-decision-facts.test.ts @@ -0,0 +1,1038 @@ +import { Compile } from "typebox/compile"; +import { afterEach, describe, expect, it } from "vitest"; +import { + AuditRunInspectResultSchema, + type DecisionReceiptV1, + type ExecutionIdentityContextV1, +} from "../../packages/gateway-protocol/src/index.js"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { activityRunInspectorSearch } from "../../ui/src/pages/activity/run-inspector-model.js"; +import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { recordAuditEvent } from "./audit-event-store.js"; +import { + pageExecutionDecisionFactsForContext, + pruneExpiredExecutionDecisionFacts, + recordExecutionDecisionFact, + summarizeExecutionDecisionFactsForContext, +} from "./execution-decision-facts.js"; +import { presentExecutionDecisionReceipts } from "./execution-decision-receipts.js"; +import { + configureExecutionIdentityAdmissionSink, + createExecutionIdentityAdmissionToken, + enqueueExecutionIdentityContextAtAdmission, + type ExecutionIdentityAdmissionEnvelope, +} from "./execution-identity-admission.js"; +import { processExecutionIdentityAdmissionWork } from "./execution-identity-context.js"; +import { bindExecutionOwnerLifecycleMetadata } from "./execution-owner-lifecycle-binding-store.js"; +import { + configureMessageActionDecisionSink, + recordMessageActionDecision, +} from "./message-action-decision.js"; +import { recordOutboundMessageProgress } from "./message-delivery-progress-store.js"; + +const RETENTION_MS = 30 * 24 * 60 * 60_000; + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function databaseOptions() { + return { env: { OPENCLAW_STATE_DIR: tempDirs.make("openclaw-decision-facts-") } }; +} + +function seedExecutionContext( + database: ReturnType, + overrides: { + runId?: string; + contextId?: string; + executionId?: string; + } = {}, +): ExecutionIdentityContextV1 { + const runId = overrides.runId ?? "run-1"; + const contextId = overrides.contextId ?? "context-1"; + const executionId = overrides.executionId ?? "execution-1"; + let envelope: ExecutionIdentityAdmissionEnvelope | undefined; + const clear = configureExecutionIdentityAdmissionSink((work) => { + if (work.kind === "capture") { + envelope = work.envelope; + } + return true; + }); + try { + enqueueExecutionIdentityContextAtAdmission( + { + runId, + agentId: "main", + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + runtime: { kind: "embedded" }, + }, + { + enabled: true, + now: 50, + contextId, + executionId, + runtimeInstanceId: "runtime-1", + }, + ); + } finally { + clear(); + } + if (!envelope) { + throw new Error("expected execution identity envelope"); + } + const stored = processExecutionIdentityAdmissionWork( + { kind: "capture", envelope }, + { ...database, now: 50 }, + ); + if ( + stored.contextId !== contextId || + stored.executionId !== executionId || + stored.runId !== runId + ) { + throw new Error(`unexpected execution context: ${JSON.stringify(stored)}`); + } + return stored; +} + +function receipt(id: string, occurredAt = 100): DecisionReceiptV1 { + return { + schemaVersion: 1, + receiptId: id, + contextId: "context-1", + executionId: "execution-1", + runId: "run-1", + actionId: `action-${id}`, + occurredAt, + action: { family: "tool", operation: "policy" }, + decision: { outcome: "denied", reasonCode: "tool_policy_denied" }, + enforcement: { + coverageState: "enforced", + evaluatorRef: "tool-policy", + policyRefs: ["tool-policy:deny"], + grantRefs: [], + contextFieldsUsed: ["runId"], + }, + source: { + owner: "tool-policy", + recordRef: `record-${id}`, + decisionBoundary: "agent-tool.before-call", + }, + missingEvidence: [], + remediation: [{ code: "choose_allowed_tool", text: "Choose an allowed tool and retry." }], + }; +} + +function tokenForContext(context: ExecutionIdentityContextV1) { + return createExecutionIdentityAdmissionToken(context.runId, { + contextId: context.contextId, + executionId: context.executionId, + now: context.createdAt, + }); +} + +describe("execution decision facts", () => { + it("persists repeated same-reason broadcast denials with opaque distinct ids", () => { + const database = databaseOptions(); + seedExecutionContext(database); + const token = createExecutionIdentityAdmissionToken("run-1", { + contextId: "context-1", + executionId: "execution-1", + now: 100, + }); + const clear = configureMessageActionDecisionSink( + (decision) => recordExecutionDecisionFact(decision, { ...database, now: 100 }) === "inserted", + ); + try { + for (const receiptDiscriminator of ["broadcast:0", "broadcast:1"]) { + expect( + recordMessageActionDecision({ + token, + actionId: "broadcast-action", + action: "broadcast", + channel: "qa-channel", + outcome: "denied", + reasonCode: "message_target_unknown", + coverageState: "enforced", + policyRefs: ["message-target:known"], + summary: "Message action was denied before platform delivery.", + remediation: [], + receiptDiscriminator, + occurredAt: 100, + }), + ).toBe(true); + } + } finally { + clear(); + } + + const receipts = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 10, + now: 100, + database, + }).receipts; + expect(receipts).toHaveLength(2); + expect(new Set(receipts.map((item) => item.receiptId)).size).toBe(2); + expect(JSON.stringify(receipts)).not.toContain("broadcast:0"); + expect(JSON.stringify(receipts)).not.toContain("broadcast:1"); + }); + + it("projects owner-native outbound delivery into run inspection", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + const now = Date.now(); + const storedEvent = recordAuditEvent( + { + sourceId: "message:outbound:queue:delivery-1:payload:0", + sourceSequence: 1, + occurredAt: now, + kind: "message", + action: "message.outbound.finished", + status: "succeeded", + outcome: "sent", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "run-1", + executionIdentityToken: tokenForContext(context), + direction: "outbound", + channel: "qa-channel", + conversationKind: "direct", + resultCount: 1, + targetId: "raw-target", + messageId: "raw-message-id", + }, + database, + ); + if (!storedEvent) { + throw new Error("expected owner-native message event"); + } + + const inspection = presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now }, + }); + expect(inspection.decisions).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + action: expect.objectContaining({ family: "message", operation: "send" }), + decision: { outcome: "allowed", reasonCode: "message_delivered" }, + enforcement: expect.objectContaining({ coverageState: "attribution-only" }), + source: expect.objectContaining({ owner: "audit_events" }), + }), + ]), + ); + expect(inspection.decisionDisplays).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + action: expect.objectContaining({ family: "message", operation: "send" }), + decision: { outcome: "allowed", reasonCode: "message_delivered" }, + provenance: { state: "verified", producer: "message-delivery" }, + }), + ]), + ); + const messageReceipt = inspection.decisions.find((item) => item.action.family === "message"); + const messageDisplay = inspection.decisionDisplays.find( + (item) => + item.provenance.state === "verified" && item.provenance.producer === "message-delivery", + ); + expect(messageDisplay?.selectorId).toBe( + `message-decision:${2_000_000_000_000 + storedEvent.sequence}`, + ); + expect(messageDisplay?.selectorId).not.toBe(messageReceipt?.receiptId); + const receiptSearch = activityRunInspectorSearch( + { kind: "run", id: context.runId }, + { id: messageDisplay?.selectorId ?? "" }, + ); + expect(new URLSearchParams(receiptSearch.slice(1)).get("receipt")).toBe( + messageDisplay?.selectorId, + ); + expect(receiptSearch).not.toContain(encodeURIComponent(messageReceipt?.receiptId ?? "")); + }); + + it("projects exact-bound cron, task, and flow owner rows without generic facts", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + const db = openOpenClawStateDatabase(database).db; + db.prepare( + `INSERT INTO cron_run_receipts ( + receipt_id, store_key, job_id, config_revision, agent_id, request_run_id, + status, owner_pid, started_at_ms, finished_at_ms + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + "cron-receipt-1", + "default", + "job-1", + "revision-1", + "main", + context.runId, + "ok", + 1, + 60, + 70, + ); + db.prepare( + `INSERT INTO task_runs ( + task_id, runtime, owner_key, scope_kind, task, status, delivery_status, + notify_policy, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + "task-1", + "cron", + "owner-1", + "session", + "private task text", + "succeeded", + "not-requested", + "never", + 61, + ); + db.prepare( + `INSERT INTO flow_runs ( + flow_id, owner_key, status, notify_policy, goal, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?)`, + ).run("flow-1", "owner-1", "succeeded", "never", "private flow goal", 62, 70); + for (const [ownerKind, ownerId] of [ + ["cron", "cron-receipt-1"], + ["task", "task-1"], + ["flow", "flow-1"], + ] as const) { + expect( + bindExecutionOwnerLifecycleMetadata({ + db, + ownerKind, + ownerId, + binding: { contextId: context.contextId, executionId: context.executionId }, + }), + ).toBe("bound"); + } + + const result = presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now: 100 }, + }); + expect(result.decisions.map((item) => item.source.owner)).toEqual([ + "agent-command", + "cron_run_receipts", + "task_runs", + "flow_runs", + ]); + expect(JSON.stringify(result.decisions)).not.toContain("private task text"); + expect(JSON.stringify(result.decisions)).not.toContain("private flow goal"); + expect(tableExists(db, "execution_decision_facts")).toBe(false); + }); + + it("does not assign run-only delivery evidence to either exact execution sharing a run id", () => { + const database = databaseOptions(); + const first = seedExecutionContext(database, { + runId: "shared-run", + contextId: "context-first", + executionId: "execution-first", + }); + const second = seedExecutionContext(database, { + runId: "shared-run", + contextId: "context-second", + executionId: "execution-second", + }); + const now = Date.now(); + recordAuditEvent( + { + sourceId: "message:shared-run:unbound", + sourceSequence: 1, + occurredAt: now, + kind: "message", + action: "message.outbound.finished", + status: "succeeded", + outcome: "sent", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "shared-run", + direction: "outbound", + channel: "qa-channel", + conversationKind: "direct", + resultCount: 1, + }, + database, + ); + + for (const context of [first, second]) { + expect( + presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now }, + }).decisions.filter((item) => item.action.family === "message"), + ).toEqual([]); + } + expect( + tableExists(openOpenClawStateDatabase(database).db, "outbound_message_execution_bindings"), + ).toBe(false); + + recordAuditEvent( + { + sourceId: "message:shared-run:first-execution", + sourceSequence: 2, + occurredAt: now + 1, + kind: "message", + action: "message.outbound.finished", + status: "succeeded", + outcome: "sent", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "shared-run", + executionIdentityToken: tokenForContext(first), + direction: "outbound", + channel: "qa-channel", + conversationKind: "direct", + resultCount: 1, + }, + database, + ); + const messageReceipts = (context: ExecutionIdentityContextV1) => + presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now: now + 1 }, + }).decisions.filter((item) => item.action.family === "message"); + expect(messageReceipts(first)).toHaveLength(1); + expect(messageReceipts(second)).toEqual([]); + expect( + openOpenClawStateDatabase(database) + .db.prepare( + "SELECT context_id, execution_id, run_id FROM outbound_message_execution_bindings", + ) + .all(), + ).toEqual([ + { + context_id: "context-first", + execution_id: "execution-first", + run_id: "shared-run", + }, + ]); + }); + + it("keeps delivery stages distinct, redacted, replay-safe, and retention bounded", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + const now = Date.now(); + const common = { + sourceSequence: 1, + occurredAt: now, + kind: "message" as const, + actorType: "agent" as const, + actorId: "main", + agentId: "main", + runId: "run-1", + executionIdentityToken: tokenForContext(context), + direction: "outbound" as const, + channel: "qa-channel", + conversationKind: "direct" as const, + targetId: "raw-channel-target", + }; + const events = [ + { + ...common, + occurredAt: now, + sourceId: "queue-1:queued", + action: "message.outbound.queued" as const, + status: "started" as const, + outcome: "queued" as const, + }, + { + ...common, + occurredAt: now + 1, + sourceId: "queue-1:platform", + action: "message.outbound.platform-started" as const, + status: "started" as const, + outcome: "platform_started" as const, + }, + { + ...common, + occurredAt: now + 2, + sourceId: "queue-1:finished", + action: "message.outbound.finished" as const, + status: "succeeded" as const, + outcome: "sent" as const, + messageId: "raw-platform-message", + }, + { + ...common, + occurredAt: now + 3, + sourceId: "queue-2:failed", + action: "message.outbound.finished" as const, + status: "failed" as const, + outcome: "failed" as const, + errorCode: "message_delivery_failed" as const, + failureStage: "queue" as const, + }, + { + ...common, + occurredAt: now + 4, + sourceId: "queue-3:failed", + action: "message.outbound.finished" as const, + status: "failed" as const, + outcome: "failed" as const, + errorCode: "message_delivery_failed" as const, + failureStage: "platform_send" as const, + }, + { + ...common, + occurredAt: now + 5, + sourceId: "queue-4:suppressed", + action: "message.outbound.finished" as const, + status: "blocked" as const, + outcome: "suppressed" as const, + reasonCode: "no_visible_payload" as const, + }, + ]; + for (const event of events) { + expect( + event.action === "message.outbound.finished" + ? recordAuditEvent(event, database) + : recordOutboundMessageProgress(event, database), + ).toBeDefined(); + } + expect( + recordOutboundMessageProgress( + { + ...common, + occurredAt: now, + sourceId: "queue-1:queued", + action: "message.outbound.queued", + status: "started", + outcome: "queued", + }, + database, + ), + ).toBeUndefined(); + + const inspect = () => + presentExecutionDecisionReceipts({ + context, + decisionCursor: "m:0:0", + decisionLimit: 10, + options: { ...database, now }, + }); + const firstInspection = inspect(); + expect(firstInspection.decisions.map((item) => item.decision.reasonCode)).toEqual([ + "message_queued", + "message_platform_started", + "message_delivered", + "message_delivery_failed_queue", + "message_delivery_failed_platform_send", + "message_suppressed_no_visible_payload", + ]); + expect(firstInspection.decisions.map((item) => item.enforcement.coverageState)).toEqual( + Array.from({ length: 6 }, () => "attribution-only"), + ); + expect(firstInspection.decisions.map((item) => item.source.owner)).toEqual([ + "outbound_message_progress", + "outbound_message_progress", + "audit_events", + "audit_events", + "audit_events", + "audit_events", + ]); + const selectors = firstInspection.decisionDisplays.map((item) => item.selectorId); + expect(selectors).toHaveLength(firstInspection.decisions.length); + expect(new Set(selectors).size).toBe(selectors.length); + expect(selectors.every((selector) => selector.startsWith("message-decision:"))).toBe(true); + expect(JSON.stringify(firstInspection)).not.toContain("raw-channel-target"); + expect(JSON.stringify(firstInspection)).not.toContain("raw-platform-message"); + + closeOpenClawStateDatabaseForTest(); + expect(inspect().decisionDisplays.map((item) => item.selectorId)).toEqual(selectors); + expect( + presentExecutionDecisionReceipts({ + context, + decisionCursor: "m:0:0", + decisionLimit: 10, + options: { ...database, now: now + RETENTION_MS + events.length + 1 }, + }).decisions, + ).toEqual([]); + }); + + it("stays absent until a future owner writes one immutable fact", () => { + const database = databaseOptions(); + seedExecutionContext(database); + const opened = openOpenClawStateDatabase(database); + expect(tableExists(opened.db, "execution_decision_facts")).toBe(false); + expect(pruneExpiredExecutionDecisionFacts({ database })).toBe(0); + expect(tableExists(opened.db, "execution_decision_facts")).toBe(false); + + expect(recordExecutionDecisionFact(receipt("receipt-1"), { ...database, now: 100 })).toBe( + "inserted", + ); + expect(recordExecutionDecisionFact(receipt("receipt-1"), { ...database, now: 100 })).toBe( + "existing", + ); + expect(() => + recordExecutionDecisionFact( + { ...receipt("receipt-1"), decision: { outcome: "allowed", reasonCode: "changed" } }, + { ...database, now: 100 }, + ), + ).toThrow("conflicts with retained state"); + + expect( + pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 10, + now: 100, + database, + }).receipts, + ).toEqual([receipt("receipt-1")]); + expect( + summarizeExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + now: 100, + database, + }), + ).toEqual({ count: 1, coverageState: "enforced", missingEvidence: [] }); + }); + + it("rejects approval duplication before creating the generic table", () => { + const database = databaseOptions(); + expect(() => + recordExecutionDecisionFact( + { + ...receipt("approval-duplicate"), + source: { + owner: "operator_approvals", + recordRef: "approval-ref", + decisionBoundary: "gateway.operator-approval.first-answer", + }, + }, + { ...database, now: 100 }, + ), + ).toThrow("owner-native table"); + expect(tableExists(openOpenClawStateDatabase(database).db, "execution_decision_facts")).toBe( + false, + ); + }); + + it("keeps high-cardinality summary work bounded and conservative", () => { + const database = databaseOptions(); + seedExecutionContext(database); + for (let index = 0; index < 130; index += 1) { + recordExecutionDecisionFact(receipt(`bounded-${String(index).padStart(3, "0")}`), { + ...database, + now: 100, + limits: { maxRows: 1_000, pruneBatchRows: 10 }, + }); + } + + expect( + summarizeExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + now: 100, + database, + }), + ).toEqual({ + count: 129, + coverageState: "unknown", + missingEvidence: ["decision.fact.summary_bounded"], + }); + }); + + it("pages equal-time facts by a bounded row key", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + for (const id of ["same-time-a", "same-time-b", "same-time-c"]) { + recordExecutionDecisionFact(receipt(id, 100), { ...database, now: 100 }); + } + + const first = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 1, + now: 100, + database, + }); + expect(first.receipts.map((item) => item.receiptId)).toEqual(["same-time-a"]); + expect(first.entries).toEqual([ + { receipt: receipt("same-time-a", 100), selectorId: "decision-fact:1" }, + ]); + expect(first.nextCursor).toEqual({ occurredAt: 100, rowId: expect.any(Number) }); + const second = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + after: first.nextCursor, + limit: 2, + now: 100, + database, + }); + expect(second.receipts.map((item) => item.receiptId)).toEqual(["same-time-b", "same-time-c"]); + expect(second.entries.map((entry) => entry.selectorId)).toEqual([ + "decision-fact:2", + "decision-fact:3", + ]); + expect( + pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 1, + now: 100, + database, + }).entries, + ).toEqual(first.entries); + expect( + new Set([...first.entries, ...second.entries].map((entry) => entry.selectorId)).size, + ).toBe(3); + + for (const decisionCursor of ["1", "001"]) { + const legacyPage = presentExecutionDecisionReceipts({ + context, + decisionCursor, + decisionLimit: 1, + options: { ...database, now: 100 }, + }); + expect(legacyPage.decisions.map((item) => item.receiptId)).toEqual(["same-time-a"]); + expect(legacyPage.decisionDisplays?.map((item) => item.selectorId)).toEqual([ + "decision-fact:1", + ]); + expect(legacyPage.nextDecisionCursor).toMatch(/^g:/); + } + const legacyPage = presentExecutionDecisionReceipts({ + context, + decisionCursor: "1", + decisionLimit: 1, + options: { ...database, now: 100 }, + }); + const next = presentExecutionDecisionReceipts({ + context, + decisionCursor: legacyPage.nextDecisionCursor, + decisionLimit: 2, + options: { ...database, now: 100 }, + }); + expect(next.decisions.map((item) => item.receiptId)).toEqual(["same-time-b", "same-time-c"]); + expect(next.decisionDisplays?.map((item) => item.selectorId)).toEqual([ + "decision-fact:2", + "decision-fact:3", + ]); + }); + + it("replaces unverified aggregate evidence with a fixed display marker", () => { + const database = databaseOptions(); + seedExecutionContext(database); + const context: ExecutionIdentityContextV1 = { + schemaVersion: 1, + contextId: "context-1", + executionId: "execution-1", + runId: "run-1", + createdAt: 50, + trustDomain: { kind: "gateway-cell", domainRef: "domain-1", state: "present" }, + invoker: { state: "absent" }, + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + agentPrincipal: { kind: "agent", domainRef: "domain-1", principalRef: "agent-main" }, + agentDefinition: { definitionRef: "main", state: "present" }, + runtimeInstance: { runtimeRef: "runtime-1", kind: "embedded", state: "present" }, + applicableGrants: [], + assurance: [], + coverageState: "unattributed", + missingEvidence: [], + }; + for (const owner of ["one", "two"] as const) { + recordExecutionDecisionFact( + { + ...receipt(owner), + missingEvidence: Array.from( + { length: 16 }, + (_, index) => `${owner}.missing.${String(index).padStart(2, "0")}`, + ), + }, + { ...database, now: 100 }, + ); + } + + const result = presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now: 100 }, + }); + expect(result.coverage).toEqual({ + state: "unknown", + missingEvidence: expect.arrayContaining(["decision.display_provenance"]), + }); + expect(JSON.stringify(result.coverage)).not.toContain("one.missing"); + expect(JSON.stringify(result.coverage)).not.toContain("two.missing"); + expect(Compile(AuditRunInspectResultSchema).Check(result)).toBe(false); + }); + + it("keeps receipt-controlled prose out of the owner display projection", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + const receiptIdSecret = "U2_R6_RECEIPT_ID_SECRET_42b17d"; + const summarySecret = "U2_R6_SUMMARY_SECRET_7c4f9a"; + const remediationCodeSecret = "U2_R6_CODE_SECRET_81d2be"; + const remediationTextSecret = "U2_R6_TEXT_SECRET_36a5c0"; + const policyRefSecret = "U2_R6_POLICY_REF_SECRET_ea731c"; + const grantRefSecret = "U2_R6_GRANT_REF_SECRET_b529f4"; + const missingEvidenceSecret = "U2_R6_MISSING_EVIDENCE_SECRET_2d97c1"; + recordExecutionDecisionFact( + { + ...receipt(receiptIdSecret), + action: { + family: "tool", + operation: "policy", + summary: summarySecret, + }, + source: { + owner: "audit_events", + recordRef: "forged-core-looking-record", + decisionBoundary: "message.outbound.finished", + }, + enforcement: { + coverageState: "enforced", + evaluatorRef: "forged-core-looking-evaluator", + policyRefs: [policyRefSecret], + grantRefs: [grantRefSecret], + contextFieldsUsed: ["runId"], + }, + missingEvidence: [missingEvidenceSecret], + remediation: [{ code: remediationCodeSecret, text: remediationTextSecret }], + }, + { ...database, now: 100 }, + ); + + const result = presentExecutionDecisionReceipts({ + context, + decisionLimit: 10, + options: { ...database, now: 100 }, + }); + expect(result.decisionDisplays).toBeDefined(); + const displayJson = JSON.stringify(result.decisionDisplays); + expect(displayJson).not.toContain(receiptIdSecret); + expect(displayJson).not.toContain(summarySecret); + expect(displayJson).not.toContain(remediationCodeSecret); + expect(displayJson).not.toContain(remediationTextSecret); + expect(displayJson).not.toContain(policyRefSecret); + expect(displayJson).not.toContain(grantRefSecret); + expect(JSON.stringify(result.coverage)).not.toContain(missingEvidenceSecret); + expect(result.coverage).toEqual({ + state: "unknown", + missingEvidence: expect.arrayContaining(["decision.display_provenance"]), + }); + expect(JSON.stringify(result.decisions)).toContain(policyRefSecret); + expect(JSON.stringify(result.decisions)).toContain(grantRefSecret); + expect(result.decisionDisplays).toEqual([ + expect.objectContaining({ + action: expect.objectContaining({ + summary: "Run admission was recorded without an identity-aware policy or grant decision.", + }), + provenance: { state: "verified", producer: "run-admission" }, + }), + expect.objectContaining({ + selectorId: "decision-fact:1", + action: { family: "decision", operation: "record" }, + decision: { outcome: "unknown", reasonCode: "decision_fact_display_unverified" }, + enforcement: { + coverageState: "unknown", + policyCount: 0, + grantCount: 0, + contextFieldsUsed: [], + }, + provenance: { state: "unverified" }, + missingEvidence: ["decision.display_provenance"], + remediation: [], + }), + ]); + expect(Compile(AuditRunInspectResultSchema).Check(result)).toBe(false); + }); + + it("rejects a generic fact whose context, execution, and run tuple is not exact", () => { + const database = databaseOptions(); + seedExecutionContext(database); + expect(() => + recordExecutionDecisionFact( + { ...receipt("wrong-execution"), executionId: "execution-2" }, + database, + ), + ).toThrow("exact retained execution context"); + expect(tableExists(openOpenClawStateDatabase(database).db, "execution_decision_facts")).toBe( + false, + ); + }); + + it("projects a fact as unknown when the requested tuple does not match", () => { + const database = databaseOptions(); + seedExecutionContext(database); + recordExecutionDecisionFact(receipt("tuple-mismatch"), { ...database, now: 100 }); + + const page = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-2", runId: "run-1" }, + limit: 10, + now: 100, + database, + }); + expect(page.entries).toEqual([ + { + selectorId: "decision-fact:1", + receipt: expect.objectContaining({ + decision: { outcome: "unknown", reasonCode: "decision_fact_execution_link_mismatch" }, + enforcement: expect.objectContaining({ coverageState: "unknown" }), + missingEvidence: ["decision.execution_link"], + }), + }, + ]); + }); + + it("enforces the 30-day read boundary and bounded retention pruning", () => { + const database = databaseOptions(); + seedExecutionContext(database); + recordExecutionDecisionFact(receipt("old", 0), { ...database, now: 0 }); + recordExecutionDecisionFact(receipt("new", RETENTION_MS + 1), { + ...database, + now: RETENTION_MS + 1, + limits: { maxRows: 10, pruneBatchRows: 1 }, + }); + + expect( + pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 10, + now: RETENTION_MS + 1, + database, + }).receipts.map((item) => item.receiptId), + ).toEqual(["new"]); + expect( + openOpenClawStateDatabase(database) + .db.prepare("SELECT COUNT(*) AS count FROM execution_decision_facts") + .get(), + ).toEqual({ count: 1 }); + }); + + it("caps retained facts without accepting a non-identical receipt id", () => { + const database = databaseOptions(); + seedExecutionContext(database); + for (const [index, id] of ["one", "two", "three"].entries()) { + recordExecutionDecisionFact(receipt(id, 100 + index), { + ...database, + now: 100 + index, + limits: { maxRows: 2, pruneBatchRows: 1 }, + }); + } + expect( + pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 10, + now: 200, + database, + }).receipts.map((item) => item.receiptId), + ).toEqual(["two", "three"]); + }); + + it("turns corrupt retained payloads into bounded unknown receipts", () => { + const database = databaseOptions(); + seedExecutionContext(database); + const context: ExecutionIdentityContextV1 = { + schemaVersion: 1, + contextId: "context-1", + executionId: "execution-1", + runId: "run-1", + createdAt: 50, + trustDomain: { kind: "gateway-cell", domainRef: "domain-1", state: "present" }, + invoker: { state: "absent" }, + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + agentPrincipal: { kind: "agent", domainRef: "domain-1", principalRef: "agent-main" }, + agentDefinition: { definitionRef: "main", state: "present" }, + runtimeInstance: { runtimeRef: "runtime-1", kind: "embedded", state: "present" }, + applicableGrants: [], + assurance: [], + coverageState: "unattributed", + missingEvidence: [], + }; + recordExecutionDecisionFact(receipt("corrupt"), { ...database, now: 100 }); + openOpenClawStateDatabase(database) + .db.prepare("UPDATE execution_decision_facts SET receipt_json = ? WHERE receipt_id = ?") + .run("{", "corrupt"); + + const page = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 10, + now: 100, + database, + }); + expect(page.entries).toEqual([ + { + selectorId: "decision-fact:1", + receipt: expect.objectContaining({ + receiptId: "corrupt", + decision: { outcome: "unknown", reasonCode: "decision_fact_record_corrupt" }, + enforcement: expect.objectContaining({ coverageState: "unknown" }), + missingEvidence: ["decision.fact.valid"], + }), + }, + ]); + expect( + summarizeExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + now: 100, + database, + }), + ).toEqual({ + count: 1, + coverageState: "unknown", + missingEvidence: ["decision.fact.valid"], + }); + expect( + presentExecutionDecisionReceipts({ + context, + decisionCursor: "g:0:0", + decisionLimit: 10, + options: { ...database, now: 100 }, + }), + ).toMatchObject({ + coverage: { + state: "unknown", + missingEvidence: expect.arrayContaining(["decision.display_provenance"]), + }, + decisions: [{ decision: { outcome: "unknown", reasonCode: "decision_fact_record_corrupt" } }], + decisionDisplays: [{ selectorId: "decision-fact:1" }], + }); + }); + + it("does not materialize an oversized retained fact payload", () => { + const database = databaseOptions(); + const context = seedExecutionContext(database); + recordExecutionDecisionFact(receipt("oversized"), { ...database, now: 100 }); + const db = openOpenClawStateDatabase(database).db; + db.exec("PRAGMA ignore_check_constraints = ON"); + db.prepare("UPDATE execution_decision_facts SET receipt_json = ? WHERE receipt_id = ?").run( + "x".repeat(20_000), + "oversized", + ); + + const page = pageExecutionDecisionFactsForContext({ + context: { contextId: "context-1", executionId: "execution-1", runId: "run-1" }, + limit: 1, + now: 100, + database, + }); + expect(page.entries).toEqual([ + { + selectorId: "decision-fact:1", + receipt: expect.objectContaining({ + decision: { outcome: "unknown", reasonCode: "decision_fact_payload_bounded" }, + missingEvidence: ["decision.fact.payload_bounded"], + }), + }, + ]); + const result = presentExecutionDecisionReceipts({ + context, + decisionCursor: "g:0:0", + decisionLimit: 1, + options: { ...database, now: 100 }, + }); + expect(result.decisions).toHaveLength(1); + expect(result.decisionDisplays).toEqual([ + expect.objectContaining({ selectorId: "decision-fact:1" }), + ]); + }); +}); diff --git a/src/audit/execution-decision-receipts.ts b/src/audit/execution-decision-receipts.ts new file mode 100644 index 0000000000000000000000000000000000000000..c3495ae3209c2f1ccdc8b0774f492faa44e5cbc8 --- /dev/null +++ b/src/audit/execution-decision-receipts.ts @@ -0,0 +1,492 @@ +/** Bounded receipt projection across admission, owner-native, and generic decision facts. */ +import type { + AuditRunInspectResult, + DecisionReceiptDisplayV1, + DecisionReceiptV1, + ExecutionIdentityContextV1, +} from "../../packages/gateway-protocol/src/index.js"; +import { + pageOperatorApprovalReceiptsForRun, + summarizeOperatorApprovalReceiptsForRun, +} from "../gateway/operator-approval-store.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { parsePositiveAuditCursor } from "./audit-cursor.js"; +import { + pageExecutionDecisionFactsForContext, + summarizeExecutionDecisionFactsForContext, +} from "./execution-decision-facts.js"; +import { + pageOwnerLifecycleReceipts, + summarizeOwnerLifecycleReceipts, + type OwnerLifecycleCursor, + type OwnerLifecycleStage, +} from "./execution-owner-lifecycle-receipts.js"; +import { + pageMessageDeliveryReceiptsForRun, + summarizeMessageDeliveryReceiptsForRun, +} from "./message-delivery-receipts.js"; + +type ExecutionDecisionReadOptions = OpenClawStateDatabaseOptions & { now?: number }; + +export type InternalAuditRunInspectResult = AuditRunInspectResult & { + decisions: DecisionReceiptV1[]; +}; + +type ProvenancedDecisionReceipt = { + receipt: DecisionReceiptV1; + provenance: DecisionReceiptDisplayV1["provenance"]; + selectorId: string; +}; + +const MAX_AGGREGATE_MISSING_EVIDENCE = 16; +const MISSING_EVIDENCE_TRUNCATED = "decision.missing_evidence_truncated"; +type DecisionStage = "approval" | "message" | "generic" | OwnerLifecycleStage; +type DecisionCursor = + | { + stage: DecisionStage; + after?: { occurredAt: number; rowId: number }; + } + | { + offset: number; + }; + +export class ExecutionDecisionCursorError extends Error { + constructor(message = "invalid execution decision cursor") { + super(message); + this.name = "ExecutionDecisionCursorError"; + } +} + +function parseDecisionCursor(value: string | undefined): DecisionCursor | undefined | null { + if (value === undefined) { + return undefined; + } + const offset = parsePositiveAuditCursor(value); + if (offset !== null && offset !== undefined) { + return { offset }; + } + const match = /^([amgctf]):(0|[1-9]\d*):(0|[1-9]\d*)$/.exec(value); + if (!match) { + return null; + } + const occurredAt = Number(match[2]); + const rowId = Number(match[3]); + if (!Number.isSafeInteger(occurredAt) || !Number.isSafeInteger(rowId)) { + return null; + } + const stage = + match[1] === "a" + ? "approval" + : match[1] === "m" + ? "message" + : match[1] === "g" + ? "generic" + : match[1] === "c" + ? "cron" + : match[1] === "t" + ? "task" + : "flow"; + return { + stage, + ...(occurredAt === 0 && rowId === 0 ? {} : { after: { occurredAt, rowId } }), + }; +} + +export function isExecutionDecisionCursor(value: string): boolean { + return parseDecisionCursor(value) !== null; +} + +function formatDecisionCursor( + stage: DecisionStage, + cursor?: { occurredAt: number; rowId: number }, +): string { + const prefix = { + approval: "a", + message: "m", + generic: "g", + cron: "c", + task: "t", + flow: "f", + }[stage]; + return `${prefix}:${cursor?.occurredAt ?? 0}:${cursor?.rowId ?? 0}`; +} + +function boundMissingEvidence(values: readonly string[]): { + missingEvidence: string[]; + truncated: boolean; +} { + const unique = [...new Set(values)].toSorted(); + if (unique.length <= MAX_AGGREGATE_MISSING_EVIDENCE) { + return { missingEvidence: unique, truncated: false }; + } + return { + missingEvidence: [ + ...unique + .filter((value) => value !== MISSING_EVIDENCE_TRUNCATED) + .slice(0, MAX_AGGREGATE_MISSING_EVIDENCE - 1), + MISSING_EVIDENCE_TRUNCATED, + ].toSorted(), + truncated: true, + }; +} + +function admissionDecision(context: ExecutionIdentityContextV1): DecisionReceiptV1 { + return { + schemaVersion: 1, + receiptId: `${context.contextId}:admission`, + contextId: context.contextId, + executionId: context.executionId, + runId: context.runId, + occurredAt: context.createdAt, + action: { + family: "run", + operation: "admission", + summary: "Run admission was recorded without an identity-aware policy or grant decision.", + }, + decision: { + outcome: "not-applicable", + reasonCode: "run_admission_identity_not_evaluated", + }, + enforcement: { + coverageState: context.coverageState, + policyRefs: [], + grantRefs: [], + contextFieldsUsed: [], + }, + source: { + owner: "agent-command", + recordRef: context.contextId, + decisionBoundary: "agent-command.run-admission", + }, + missingEvidence: [...context.missingEvidence], + remediation: [ + { + code: "no_identity_enforcement_claimed", + text: "Treat this receipt as attribution only; it does not prove authorization.", + }, + ], + }; +} + +function projectDecisionDisplay({ + receipt, + provenance, + selectorId, +}: ProvenancedDecisionReceipt): DecisionReceiptDisplayV1 { + if (provenance.state === "unverified") { + return { + schemaVersion: 1, + selectorId, + occurredAt: receipt.occurredAt, + action: { family: "decision", operation: "record" }, + decision: { outcome: "unknown", reasonCode: "decision_fact_display_unverified" }, + enforcement: { + coverageState: "unknown", + policyCount: 0, + grantCount: 0, + contextFieldsUsed: [], + }, + provenance, + missingEvidence: ["decision.display_provenance"], + remediation: [], + }; + } + const counts = { + policyCount: receipt.enforcement.policyRefs.length, + grantCount: receipt.enforcement.grantRefs.length, + }; + return { + schemaVersion: 1, + selectorId, + occurredAt: receipt.occurredAt, + action: { + family: receipt.action.family, + operation: receipt.action.operation, + ...(receipt.action.summary ? { summary: receipt.action.summary } : {}), + }, + decision: receipt.decision, + enforcement: { + coverageState: receipt.enforcement.coverageState, + ...counts, + contextFieldsUsed: receipt.enforcement.contextFieldsUsed, + }, + provenance, + missingEvidence: receipt.missingEvidence, + remediation: receipt.remediation, + }; +} + +export function presentExecutionDecisionReceipts(params: { + context: ExecutionIdentityContextV1; + decisionCursor?: string; + decisionLimit?: number; + options: ExecutionDecisionReadOptions; +}): InternalAuditRunInspectResult { + const cursor = parseDecisionCursor(params.decisionCursor); + if (cursor === null) { + throw new ExecutionDecisionCursorError(); + } + const decisionLimit = params.decisionLimit ?? 50; + const now = params.options.now ?? Date.now(); + const opaqueCursor = cursor && "stage" in cursor ? cursor : undefined; + const legacyOffset = cursor && "offset" in cursor ? cursor.offset - 1 : undefined; + const approvalSummary = summarizeOperatorApprovalReceiptsForRun({ + context: { + contextId: params.context.contextId, + executionId: params.context.executionId, + runId: params.context.runId, + }, + nowMs: now, + databaseOptions: params.options, + exactCount: legacyOffset !== undefined, + }); + const genericSummary = summarizeExecutionDecisionFactsForContext({ + context: params.context, + now, + database: params.options, + }); + const messageSummary = summarizeMessageDeliveryReceiptsForRun({ + context: params.context, + options: { ...params.options, now }, + }); + const cronSummary = summarizeOwnerLifecycleReceipts({ + stage: "cron", + context: params.context, + options: params.options, + }); + const taskSummary = summarizeOwnerLifecycleReceipts({ + stage: "task", + context: params.context, + options: params.options, + }); + const flowSummary = summarizeOwnerLifecycleReceipts({ + stage: "flow", + context: params.context, + options: params.options, + }); + const stages: Array<{ + stage: DecisionStage; + count: number; + page: (params: { after?: OwnerLifecycleCursor; offset?: number; limit: number }) => { + entries: ProvenancedDecisionReceipt[]; + nextCursor?: OwnerLifecycleCursor; + }; + }> = [ + { + stage: "approval", + count: approvalSummary.count, + page: ({ after, offset, limit }) => { + const page = pageOperatorApprovalReceiptsForRun({ + context: { + contextId: params.context.contextId, + executionId: params.context.executionId, + runId: params.context.runId, + }, + after, + offset, + limit, + nowMs: now, + databaseOptions: params.options, + }); + return { + entries: page.entries.map((entry) => ({ + receipt: entry.receipt, + provenance: { state: "verified", producer: "operator-approval" }, + selectorId: entry.selectorId, + })), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }; + }, + }, + { + stage: "message", + count: messageSummary.count, + page: ({ after, offset, limit }) => { + const page = pageMessageDeliveryReceiptsForRun({ + context: params.context, + after, + offset, + limit, + options: { ...params.options, now }, + }); + return { + entries: page.entries.map((entry) => ({ + receipt: entry.receipt, + provenance: { state: "verified", producer: "message-delivery" }, + selectorId: entry.selectorId, + })), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }; + }, + }, + { + stage: "generic", + count: genericSummary.count, + page: ({ after, offset, limit }) => { + const page = pageExecutionDecisionFactsForContext({ + context: params.context, + after, + offset, + limit, + now, + database: params.options, + }); + return { + entries: page.entries.map((entry) => ({ + receipt: entry.receipt, + provenance: { state: "unverified" }, + selectorId: entry.selectorId, + })), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }; + }, + }, + ...(["cron", "task", "flow"] as const).map((stage) => ({ + stage, + count: { cron: cronSummary, task: taskSummary, flow: flowSummary }[stage].count, + page: ({ + after, + offset, + limit, + }: { + after?: OwnerLifecycleCursor; + offset?: number; + limit: number; + }) => { + const page = pageOwnerLifecycleReceipts({ + stage, + context: params.context, + after, + offset, + limit, + options: params.options, + }); + return { + entries: page.entries.map((entry) => ({ + receipt: entry.receipt, + provenance: { state: "verified" as const, producer: entry.displayProducer }, + selectorId: entry.selectorId, + })), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }; + }, + })), + ]; + const decisions: ProvenancedDecisionReceipt[] = []; + let remainingLimit = decisionLimit; + let nextDecisionCursor: string | undefined; + + if (cursor === undefined && remainingLimit > 0) { + decisions.push({ + receipt: admissionDecision(params.context), + provenance: { state: "verified", producer: "run-admission" }, + selectorId: `${params.context.contextId}:admission`, + }); + remainingLimit -= 1; + if (remainingLimit === 0 && stages.some((stage) => stage.count > 0)) { + // The shipped first successor remains the approval cursor even when that owner is empty. + nextDecisionCursor = formatDecisionCursor("approval"); + } + } + let startStage = 0; + let firstStageOffset: number | undefined; + if (opaqueCursor) { + startStage = stages.findIndex((stage) => stage.stage === opaqueCursor.stage); + } else if (legacyOffset !== undefined) { + let preceding = 0; + startStage = stages.findIndex((stage) => { + if (legacyOffset < preceding + stage.count) { + firstStageOffset = legacyOffset - preceding; + return true; + } + preceding += stage.count; + return false; + }); + if (startStage < 0) { + startStage = stages.length; + } + } + for (let index = startStage; index < stages.length && remainingLimit > 0; index += 1) { + const stage = stages[index]; + if (!stage) { + continue; + } + let page; + try { + page = stage.page({ + ...(index === startStage && opaqueCursor?.stage === stage.stage + ? { after: opaqueCursor.after } + : {}), + ...(index === startStage && firstStageOffset !== undefined + ? { offset: firstStageOffset } + : {}), + limit: remainingLimit, + }); + } catch (error) { + if (error instanceof Error && error.message.includes("cursor is no longer retained")) { + throw new ExecutionDecisionCursorError( + "decision cursor is no longer retained; restart inspection without --cursor", + ); + } + throw error; + } + decisions.push(...page.entries); + remainingLimit -= page.entries.length; + if (page.nextCursor) { + nextDecisionCursor = formatDecisionCursor(stage.stage, page.nextCursor); + break; + } + if (remainingLimit === 0) { + const successor = stages.slice(index + 1).find((candidate) => candidate.count > 0); + nextDecisionCursor = successor ? formatDecisionCursor(successor.stage) : undefined; + } + } + const ownerCoverage = new Set( + [ + approvalSummary.coverageState, + messageSummary.coverageState, + cronSummary.coverageState, + taskSummary.coverageState, + flowSummary.coverageState, + ].filter( + (coverageState): coverageState is NonNullable => + coverageState !== undefined, + ), + ); + const hasUnverifiedGenericDecisions = genericSummary.count > 0; + const boundedEvidence = boundMissingEvidence([ + ...params.context.missingEvidence, + ...approvalSummary.missingEvidence, + ...messageSummary.missingEvidence, + ...cronSummary.missingEvidence, + ...taskSummary.missingEvidence, + ...flowSummary.missingEvidence, + ...(hasUnverifiedGenericDecisions ? ["decision.display_provenance"] : []), + ]); + const coverageState = boundedEvidence.truncated + ? "unknown" + : hasUnverifiedGenericDecisions + ? "unknown" + : ownerCoverage.has("unsupported") + ? "unsupported" + : ownerCoverage.has("unknown") + ? "unknown" + : ownerCoverage.has("enforced") + ? "enforced" + : ownerCoverage.has("attribution-only") + ? "attribution-only" + : params.context.coverageState; + return { + schemaVersion: 1, + run: { + runId: params.context.runId, + executionId: params.context.executionId, + status: "known", + }, + identity: { state: "present", context: params.context }, + decisions: decisions.map(({ receipt }) => receipt), + decisionDisplays: decisions.map(projectDecisionDisplay), + coverage: { state: coverageState, missingEvidence: boundedEvidence.missingEvidence }, + ...(nextDecisionCursor ? { nextDecisionCursor } : {}), + }; +} diff --git a/src/audit/execution-decision-work.test.ts b/src/audit/execution-decision-work.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..e90ac8519841cbd70ab1bc6a1dc4b76349af551b --- /dev/null +++ b/src/audit/execution-decision-work.test.ts @@ -0,0 +1,251 @@ +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { createAuditEventWriter } from "./audit-event-writer.js"; +import { pageExecutionDecisionFactsForContext } from "./execution-decision-facts.js"; +import type { ExecutionDecisionWork } from "./execution-decision-work.js"; +import { + configureExecutionIdentityAdmissionSink, + createExecutionIdentityAdmissionToken, + enqueueExecutionIdentityContextAtAdmission, +} from "./execution-identity-admission.js"; + +function decisionWork(params: { + token: ReturnType; + receiptId: string; + rawResource: string; + rawTarget: string; + targetNamespace?: "model-route" | "session"; +}): ExecutionDecisionWork { + return { + workVersion: 1, + token: params.token, + receipt: { + schemaVersion: 1, + receiptId: params.receiptId, + occurredAt: params.token.createdAt + 1, + action: { + family: "model-routing", + operation: "select", + summary: "Selected one admitted model route.", + }, + decision: { outcome: "allowed", reasonCode: "model_route_selected" }, + enforcement: { + coverageState: "attribution-only", + policyRefs: [], + grantRefs: [], + contextFieldsUsed: ["contextId", "executionId", "runId"], + }, + source: { + owner: "model-routing", + recordRef: params.receiptId, + decisionBoundary: "agent-runtime.post-admission", + }, + missingEvidence: [], + remediation: [], + }, + refs: { + resource: { namespace: "credential-profile", value: params.rawResource }, + target: { namespace: params.targetNamespace ?? "model-route", value: params.rawTarget }, + }, + }; +} + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("private execution decision work", () => { + it.each(["raw ref", "complete envelope"] as const)( + "rejects an oversized %s before the FIFO clone boundary", + async (oversizedPart) => { + const stateDir = tempDirs.make("openclaw-audit-private-decision-bounds-"); + const errors: string[] = []; + const writer = createAuditEventWriter({ stateDir, onError: (error) => errors.push(error) }); + const token = createExecutionIdentityAdmissionToken("bounded-private-decision-run", { + contextId: "bounded-private-decision-context", + executionId: "bounded-private-decision-execution", + now: 1_000, + }); + const work = decisionWork({ + token, + receiptId: "bounded-private-decision", + rawResource: oversizedPart === "raw ref" ? "r".repeat(4_097) : "resource", + rawTarget: "target", + }); + if (oversizedPart === "complete envelope") { + const refs = Array.from({ length: 16 }, (_, index) => `${index}:`.padEnd(256, "p")); + work.receipt.enforcement.policyRefs = refs; + work.receipt.enforcement.grantRefs = refs; + work.receipt.enforcement.contextFieldsUsed = refs; + work.receipt.missingEvidence = refs; + work.receipt.remediation = Array.from({ length: 8 }, (_, index) => ({ + code: `repair-${index}`, + text: "r".repeat(512), + })); + } + + await writer.ready; + const clone = vi.spyOn(globalThis, "structuredClone"); + expect(writer.recordExecutionDecisionWork(work)).toBe(false); + expect(clone).not.toHaveBeenCalled(); + clone.mockRestore(); + await writer.stop(); + + expect(errors).toEqual(["audit execution decision receipt could not be queued"]); + }, + ); + + it("projects private refs inside the admission FIFO without retaining raw owners", async () => { + const stateDir = tempDirs.make("openclaw-audit-private-decision-"); + const database = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const errors: string[] = []; + const writer = createAuditEventWriter({ stateDir, onError: (error) => errors.push(error) }); + const admittedAt = Date.now(); + const token = createExecutionIdentityAdmissionToken("private-decision-run", { + contextId: "private-decision-context", + executionId: "private-decision-execution", + now: admittedAt, + }); + const rawCredentialProfile = "openai:user@example.test"; + const rawModelTarget = "openai\0gpt-5.6-sol"; + const clearAdmissionSink = configureExecutionIdentityAdmissionSink( + writer.recordExecutionIdentity, + ); + + await writer.ready; + expect( + enqueueExecutionIdentityContextAtAdmission( + { + runId: token.runId, + agentId: "main", + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + runtime: { kind: "embedded" }, + }, + { enabled: true, token, runtimeInstanceId: "private-decision-runtime" }, + )?.accepted, + ).toBe(true); + expect( + writer.recordExecutionDecisionWork( + decisionWork({ + token, + receiptId: "private-model-route", + rawResource: rawCredentialProfile, + rawTarget: rawModelTarget, + }), + ), + ).toBe(true); + clearAdmissionSink(); + await writer.stop(); + + expect(errors).toEqual([]); + const [receipt] = pageExecutionDecisionFactsForContext({ + context: token, + limit: 10, + now: admittedAt + 1, + database, + }).receipts; + expect(receipt).toMatchObject({ + contextId: token.contextId, + executionId: token.executionId, + runId: token.runId, + action: { + family: "model-routing", + resourceRef: expect.stringMatching(/^hmac-sha256:v1:/u), + targetRef: expect.stringMatching(/^hmac-sha256:v1:/u), + }, + }); + expect(receipt?.action.resourceRef).not.toBe(receipt?.action.targetRef); + const sqlite = new DatabaseSync(path.join(stateDir, "state", "openclaw.sqlite"), { + readOnly: true, + }); + try { + const persisted = JSON.stringify( + sqlite + .prepare("SELECT * FROM execution_decision_facts WHERE receipt_id = ?") + .get("private-model-route"), + ); + expect(persisted).not.toContain(rawCredentialProfile); + expect(persisted).not.toContain(rawModelTarget); + } finally { + sqlite.close(); + } + }); + + it("keeps refs restart-stable, namespace-separated, and installation-local", async () => { + const rawRef = "same-private-owner-ref"; + const firstStateDir = tempDirs.make("openclaw-audit-private-stability-"); + const write = async (params: { + stateDir: string; + suffix: string; + targetNamespace?: "model-route" | "session"; + }) => { + const now = Date.now(); + const token = createExecutionIdentityAdmissionToken(`run-${params.suffix}`, { + contextId: `context-${params.suffix}`, + executionId: `execution-${params.suffix}`, + now, + }); + const database = { env: { OPENCLAW_STATE_DIR: params.stateDir } }; + const writer = createAuditEventWriter({ stateDir: params.stateDir }); + const clearAdmissionSink = configureExecutionIdentityAdmissionSink( + writer.recordExecutionIdentity, + ); + await writer.ready; + expect( + enqueueExecutionIdentityContextAtAdmission( + { + runId: token.runId, + agentId: "main", + ingress: { kind: "local-cli", boundary: "agent-command.local" }, + runtime: { kind: "embedded" }, + }, + { enabled: true, token, runtimeInstanceId: `runtime-${params.suffix}` }, + )?.accepted, + ).toBe(true); + expect( + writer.recordExecutionDecisionWork( + decisionWork({ + token, + receiptId: `receipt-${params.suffix}`, + rawResource: rawRef, + rawTarget: rawRef, + ...(params.targetNamespace ? { targetNamespace: params.targetNamespace } : {}), + }), + ), + ).toBe(true); + clearAdmissionSink(); + await writer.stop(); + return pageExecutionDecisionFactsForContext({ + context: token, + limit: 1, + now: now + 1, + database, + }).receipts[0]!; + }; + + const first = await write({ stateDir: firstStateDir, suffix: "first" }); + closeOpenClawStateDatabaseForTest(); + const restarted = await write({ stateDir: firstStateDir, suffix: "restart" }); + const sessionScoped = await write({ + stateDir: firstStateDir, + suffix: "session", + targetNamespace: "session", + }); + closeOpenClawStateDatabaseForTest(); + const otherInstallation = await write({ + stateDir: tempDirs.make("openclaw-audit-private-other-installation-"), + suffix: "other", + }); + + expect(restarted.action.resourceRef).toBe(first.action.resourceRef); + expect(restarted.action.targetRef).toBe(first.action.targetRef); + expect(first.action.resourceRef).not.toBe(first.action.targetRef); + expect(sessionScoped.action.targetRef).not.toBe(first.action.targetRef); + expect(otherInstallation.action.resourceRef).not.toBe(first.action.resourceRef); + expect(otherInstallation.action.targetRef).not.toBe(first.action.targetRef); + }); +}); diff --git a/src/audit/execution-identity-context-build.ts b/src/audit/execution-identity-context-build.ts new file mode 100644 index 0000000000000000000000000000000000000000..a9193ddf798394772c27e8457547989a045ff9fc --- /dev/null +++ b/src/audit/execution-identity-context-build.ts @@ -0,0 +1,195 @@ +/** Worker-only canonical context construction and bounded value helpers. */ +import type { DatabaseSync } from "node:sqlite"; +import type { ExecutionIdentityContextV1 } from "../../packages/gateway-protocol/src/index.js"; +import { validateExecutionIdentityContextV1 } from "../../packages/gateway-protocol/src/index.js"; +import { pseudonymizeExecutionIdentityRef } from "./audit-identity.js"; +import type { ExecutionIdentityAdmissionEnvelope } from "./execution-identity-admission.js"; +import { sortUniqueExecutionIdentityEntries } from "./execution-identity-ordering.js"; +import { executionIdentitySpawnAdmission } from "./execution-identity-spawn-admission.js"; + +const EXECUTION_IDENTITY_CONTEXT_MAX_BYTES = 16 * 1024; + +export function ensureBoundedExecutionIdentityRef( + value: string, + label: string, + maxLength = 256, +): string { + if (!value || value.length > maxLength) { + throw new Error(`${label} must be between 1 and ${String(maxLength)} characters`); + } + return value; +} + +function ensureRawRef(value: string, label: string): string { + return ensureBoundedExecutionIdentityRef(value, label, 4_096); +} + +export function freezeExecutionIdentityContext(value: T, seen = new WeakSet()): T { + if (!value || typeof value !== "object" || seen.has(value as object)) { + return value; + } + seen.add(value as object); + for (const nested of Object.values(value as Record)) { + freezeExecutionIdentityContext(nested, seen); + } + return Object.freeze(value); +} + +function hmacRef( + db: DatabaseSync, + kind: Parameters[0]["kind"], + scope: string, + value: string, +): string { + return pseudonymizeExecutionIdentityRef({ + db, + kind, + scope: ensureBoundedExecutionIdentityRef(scope, "HMAC scope"), + value: ensureRawRef(value, "HMAC value"), + }); +} + +export function buildExecutionIdentityContext( + db: DatabaseSync, + envelope: ExecutionIdentityAdmissionEnvelope, + fixed: { contextId: string; createdAt: number }, +): ExecutionIdentityContextV1 { + const runId = ensureBoundedExecutionIdentityRef(envelope.runId, "run id"); + const executionId = ensureBoundedExecutionIdentityRef(envelope.executionId, "execution id"); + const agentId = ensureBoundedExecutionIdentityRef(envelope.agentId, "agent id"); + const contextId = ensureBoundedExecutionIdentityRef(fixed.contextId, "context id"); + const domainRef = hmacRef(db, "domain", "gateway-cell", "gateway-cell"); + const runtimeRef = hmacRef( + db, + "runtime", + domainRef, + ensureRawRef(envelope.runtimeInstanceId, "runtime instance id"), + ); + const invoker = + envelope.invoker?.state === "present" + ? { + state: "present" as const, + principal: { + kind: envelope.invoker.kind, + domainRef, + principalRef: hmacRef( + db, + "principal", + `${domainRef}:${envelope.invoker.kind}`, + envelope.invoker.rawPrincipalRef, + ), + ...(envelope.invoker.displayLabel !== undefined + ? { displayLabel: envelope.invoker.displayLabel } + : {}), + }, + } + : envelope.invoker?.state === "unknown" + ? { state: "unknown" as const } + : { state: "absent" as const }; + const assurance = sortUniqueExecutionIdentityEntries( + envelope.assurance.map((item) => ({ + kind: item.kind, + evidenceRef: hmacRef(db, "evidence", `${domainRef}:${item.kind}`, item.rawEvidenceRef), + strength: item.strength, + })), + (item) => `${item.kind}\0${item.evidenceRef}\0${item.strength}`, + ); + const applicableGrants = sortUniqueExecutionIdentityEntries( + envelope.applicableGrants.map((grant) => ({ + grantRef: hmacRef(db, "grant", domainRef, grant.rawGrantRef), + state: grant.state, + })), + (grant) => `${grant.grantRef}\0${grant.state}`, + ); + const serializedSpawnFacts = executionIdentitySpawnAdmission({ + operation: "read", + value: envelope, + }); + const [lineageFacts, spawnMissingEvidence] = serializedSpawnFacts + ? executionIdentitySpawnAdmission({ operation: "parse", value: serializedSpawnFacts }) + : [undefined, []]; + const lineage = lineageFacts + ? { + ...(typeof lineageFacts.parentContextId === "string" + ? { parentContextId: lineageFacts.parentContextId } + : {}), + ...(typeof lineageFacts.parentExecutionId === "string" + ? { parentExecutionId: lineageFacts.parentExecutionId } + : {}), + ...(typeof lineageFacts.parentRunId === "string" + ? { parentRunId: lineageFacts.parentRunId } + : {}), + parentAgentPrincipal: { + kind: "agent" as const, + domainRef, + principalRef: lineageFacts.parentAgentId, + }, + delegationRef: hmacRef( + db, + "grant", + `${domainRef}:delegation`, + JSON.stringify([ + lineageFacts.relation, + lineageFacts.rawRequesterRef, + lineageFacts.rawControllerRef, + lineageFacts.localPolicyRefs, + lineageFacts.targetPolicyRefs, + ]), + ), + depth: lineageFacts.depth, + } + : undefined; + const missingEvidence = sortUniqueExecutionIdentityEntries( + [ + ...(envelope.invoker?.state === "present" ? [] : ["invoker.principal"]), + ...spawnMissingEvidence, + ], + (item) => item, + ); + const context: Record = { + schemaVersion: 1, + contextId, + executionId, + runId, + createdAt: fixed.createdAt, + trustDomain: { kind: "gateway-cell", domainRef, state: "present" }, + invoker, + ingress: { + kind: envelope.ingress.kind, + boundary: ensureBoundedExecutionIdentityRef(envelope.ingress.boundary, "ingress boundary"), + state: envelope.ingress.state, + ...(envelope.ingress.rawSourceRef + ? { + sourceRef: hmacRef( + db, + "principal", + `${domainRef}:ingress:${envelope.ingress.kind}`, + envelope.ingress.rawSourceRef, + ), + } + : {}), + }, + agentPrincipal: { kind: "agent", domainRef, principalRef: agentId }, + agentDefinition: { definitionRef: agentId, state: "present" }, + runtimeInstance: { runtimeRef, kind: envelope.runtime.kind, state: "present" }, + applicableGrants, + assurance, + ...(lineage ? { lineage } : {}), + coverageState: lineage + ? "attribution-only" + : envelope.invoker?.state === "present" + ? "attribution-only" + : envelope.invoker?.state === "unknown" + ? "unknown" + : "unattributed", + missingEvidence, + }; + if (!validateExecutionIdentityContextV1(context)) { + throw new Error("prepared execution identity context violates the V1 contract"); + } + const encoded = JSON.stringify(context); + if (Buffer.byteLength(encoded, "utf8") > EXECUTION_IDENTITY_CONTEXT_MAX_BYTES) { + throw new Error("prepared execution identity context exceeds 16 KiB"); + } + return freezeExecutionIdentityContext(context); +} diff --git a/src/audit/execution-identity-context.test.ts b/src/audit/execution-identity-context.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..ddac67dded80bddbb3c8685a7efd712176dd3c5b --- /dev/null +++ b/src/audit/execution-identity-context.test.ts @@ -0,0 +1,1065 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + insertOperatorApproval, + resolveOperatorApproval, +} from "../gateway/operator-approval-store.js"; +import { openNodeSqliteDatabase } from "../infra/node-sqlite.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + type OpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { recordAuditEvent } from "./audit-event-store.js"; +import { + configureExecutionIdentityAdmissionSink, + createExecutionIdentityAdmissionToken, + enqueueExecutionIdentityContextAtAdmission, + type ExecutionIdentityAdmissionEnvelope, + type ExecutionIdentityAdmissionFacts, +} from "./execution-identity-admission.js"; +import { + inspectExecutionIdentityRun, + processExecutionIdentityAdmissionWork, + pruneExpiredExecutionIdentityContexts, +} from "./execution-identity-context.js"; + +const RETENTION_MS = 30 * 24 * 60 * 60_000; + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function databaseOptions() { + return { env: { OPENCLAW_STATE_DIR: tempDirs.make("openclaw-identity-") } }; +} + +function openIndependentStateDatabase(path: string): OpenClawStateDatabase { + return { + db: openNodeSqliteDatabase(path), + path, + walMaintenance: { checkpoint: () => true, close: () => true }, + }; +} + +function facts( + runId: string, + overrides: Partial = {}, +): ExecutionIdentityAdmissionFacts { + return { + runId, + agentId: "main", + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + runtime: { kind: "embedded" }, + ...overrides, + }; +} + +function captureExecutionIdentityAdmissionEnvelope( + admissionFacts: ExecutionIdentityAdmissionFacts, + options: { + now?: number; + contextId?: string; + executionId?: string; + runtimeInstanceId?: string; + } = {}, +): ExecutionIdentityAdmissionEnvelope { + const { contextId, executionId, runtimeInstanceId, now } = options; + let envelope: ExecutionIdentityAdmissionEnvelope | undefined; + const clear = configureExecutionIdentityAdmissionSink((captured) => { + if (captured.kind === "capture") { + envelope = captured.envelope; + } + return true; + }); + try { + const result = enqueueExecutionIdentityContextAtAdmission(admissionFacts, { + enabled: true, + ...(contextId !== undefined ? { contextId } : {}), + ...(executionId !== undefined ? { executionId } : {}), + ...(runtimeInstanceId !== undefined ? { runtimeInstanceId } : {}), + ...(now !== undefined ? { now } : {}), + }); + if (!result || !envelope) { + throw new Error("expected admission envelope"); + } + return envelope; + } finally { + clear(); + } +} + +function persistExecutionIdentityAdmissionEnvelope( + envelope: ExecutionIdentityAdmissionEnvelope, + options: Parameters[1] = {}, +) { + return processExecutionIdentityAdmissionWork({ kind: "capture", envelope }, options); +} + +function prepareExecutionIdentityContextAtAdmission( + admissionFacts: ExecutionIdentityAdmissionFacts, + options: { + database?: OpenClawStateDatabase; + env?: NodeJS.ProcessEnv; + now?: number; + contextId?: string; + executionId?: string; + runtimeInstanceId?: string; + limits?: { maxRows: number; pruneBatchRows: number }; + } = {}, +) { + const { contextId, executionId, runtimeInstanceId, now, limits, ...database } = options; + const envelope = captureExecutionIdentityAdmissionEnvelope(admissionFacts, { + ...(contextId !== undefined ? { contextId } : {}), + ...(executionId !== undefined ? { executionId } : {}), + ...(runtimeInstanceId !== undefined ? { runtimeInstanceId } : {}), + ...(now !== undefined ? { now } : {}), + }); + return persistExecutionIdentityAdmissionEnvelope(envelope, { + ...database, + ...(now !== undefined ? { now } : {}), + ...(limits !== undefined ? { limits } : {}), + }); +} + +function recordDeniedApprovalForRun( + runId: string, + database: ReturnType, + id = "denied-approval", + binding?: { contextId: string; executionId: string }, +): void { + insertOperatorApproval({ + approval: { + id, + kind: "exec", + presentation: { + kind: "exec", + commandText: "details withheld", + allowedDecisions: ["allow-once", "deny"], + }, + source: { runId, toolCallId: "private-tool-call", toolName: "exec" }, + runtimeEpoch: "runtime-1", + createdAtMs: 100, + expiresAtMs: 1_000, + ...(binding + ? { + executionIdentityToken: { + tokenVersion: 1, + createdAt: 100, + runId, + contextId: binding.contextId, + executionId: binding.executionId, + }, + } + : {}), + }, + databaseOptions: database, + }); + resolveOperatorApproval({ + id, + decision: "deny", + resolver: { kind: "device", id: "private-reviewer-device" }, + nowMs: 200, + databaseOptions: database, + }); +} + +describe("execution identity context storage", () => { + it("replays one byte-identical canonical context idempotently across restart", () => { + const database = databaseOptions(); + const envelope = captureExecutionIdentityAdmissionEnvelope(facts("run-1"), { + now: 100, + contextId: "context-1", + executionId: "execution-1", + runtimeInstanceId: "runtime-secret-1", + }); + const first = persistExecutionIdentityAdmissionEnvelope(envelope, { ...database, now: 100 }); + + closeOpenClawStateDatabaseForTest(); + const second = persistExecutionIdentityAdmissionEnvelope(structuredClone(envelope), { + ...database, + now: 999, + }); + + expect(second).toEqual(first); + expect(first.coverageState).toBe("unattributed"); + expect(first.invoker).toEqual({ state: "absent" }); + expect(Object.isFrozen(first)).toBe(true); + expect(Object.isFrozen(first.runtimeInstance)).toBe(true); + expect(JSON.stringify(first)).not.toContain("runtime-secret-1"); + + closeOpenClawStateDatabaseForTest(); + const afterRestart = inspectExecutionIdentityRun( + { executionId: "execution-1" }, + { + ...database, + now: 999, + }, + ); + expect(afterRestart.identity).toEqual({ state: "present", context: first }); + }); + + it("keeps explicit unknown invoker evidence distinct from omission across restart", () => { + const database = databaseOptions(); + const unknown = prepareExecutionIdentityContextAtAdmission( + facts("run-unknown", { invoker: { state: "unknown" } }), + { + ...database, + now: 100, + contextId: "context-unknown", + executionId: "execution-unknown", + runtimeInstanceId: "runtime-unknown", + }, + ); + const absent = prepareExecutionIdentityContextAtAdmission(facts("run-absent"), { + ...database, + now: 101, + contextId: "context-absent", + executionId: "execution-absent", + runtimeInstanceId: "runtime-absent", + }); + + expect(unknown).toMatchObject({ + invoker: { state: "unknown" }, + coverageState: "unknown", + missingEvidence: ["invoker.principal"], + }); + expect(unknown.invoker).not.toHaveProperty("principal"); + expect(absent).toMatchObject({ + invoker: { state: "absent" }, + coverageState: "unattributed", + missingEvidence: ["invoker.principal"], + }); + + closeOpenClawStateDatabaseForTest(); + const unknownAfterRestart = inspectExecutionIdentityRun( + { executionId: "execution-unknown" }, + { ...database, now: 101 }, + ); + expect(unknownAfterRestart.identity).toEqual({ state: "present", context: unknown }); + expect(unknownAfterRestart.coverage).toEqual({ + state: "unknown", + missingEvidence: ["invoker.principal"], + }); + expect(unknownAfterRestart.decisions).toEqual([ + expect.objectContaining({ + enforcement: expect.objectContaining({ coverageState: "unknown" }), + missingEvidence: ["invoker.principal"], + }), + ]); + expect( + inspectExecutionIdentityRun({ executionId: "execution-absent" }, { ...database, now: 101 }) + .identity, + ).toEqual({ state: "present", context: absent }); + }); + + it.each([ + { + difference: "contextId", + mutate: (envelope: ExecutionIdentityAdmissionEnvelope) => ({ + ...envelope, + contextId: "context-conflicting", + }), + }, + { + difference: "createdAt", + mutate: (envelope: ExecutionIdentityAdmissionEnvelope) => ({ + ...envelope, + createdAt: envelope.createdAt + 1, + }), + }, + { + difference: "identity facts", + mutate: (envelope: ExecutionIdentityAdmissionEnvelope) => ({ + ...envelope, + agentId: "other", + }), + }, + ])( + "conflicts on a same-execution $difference and leaves canonical bytes unchanged", + ({ mutate }) => { + const database = databaseOptions(); + const envelope = captureExecutionIdentityAdmissionEnvelope(facts("run-conflict"), { + contextId: "context-original", + executionId: "execution-original", + now: 100, + runtimeInstanceId: "runtime-1", + }); + const original = persistExecutionIdentityAdmissionEnvelope(envelope, { + ...database, + now: 100, + }); + const originalRow = openOpenClawStateDatabase(database) + .db.prepare("SELECT context_json FROM execution_identity_contexts WHERE execution_id = ?") + .get("execution-original"); + + expect(() => + persistExecutionIdentityAdmissionEnvelope(mutate(envelope), { ...database, now: 101 }), + ).toThrow("execution identity context conflict"); + expect( + openOpenClawStateDatabase(database) + .db.prepare("SELECT context_json FROM execution_identity_contexts WHERE execution_id = ?") + .get("execution-original"), + ).toEqual(originalRow); + expect( + inspectExecutionIdentityRun( + { executionId: "execution-original" }, + { ...database, now: 101 }, + ).identity, + ).toEqual({ state: "present", context: original }); + }, + ); + + it("keeps distinct turns sharing one run correlation exactly inspectable", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("session-run"), { + ...database, + now: 100, + contextId: "context-first", + executionId: "execution-first", + runtimeInstanceId: "runtime-1", + }); + prepareExecutionIdentityContextAtAdmission(facts("session-run"), { + ...database, + now: 101, + contextId: "context-second", + executionId: "execution-second", + runtimeInstanceId: "runtime-1", + }); + recordDeniedApprovalForRun("session-run", database, "shared-run-approval", { + contextId: "context-first", + executionId: "execution-first", + }); + + const discovery = inspectExecutionIdentityRun( + { runId: "session-run" }, + { ...database, now: 300 }, + ); + expect(discovery).toMatchObject({ + run: { runId: "session-run", status: "known" }, + identity: { + state: "ambiguous", + reasonCode: "execution_selection_required", + candidates: [ + { executionId: "execution-first", contextId: "context-first" }, + { executionId: "execution-second", contextId: "context-second" }, + ], + }, + decisions: [], + }); + for (const [executionOffset, executionId, nextExecutionCursor] of [ + [0, "execution-first", "1"], + [1, "execution-second", undefined], + ] as const) { + expect( + inspectExecutionIdentityRun( + { runId: "session-run", executionOffset, executionLimit: 1 }, + { ...database, now: 300 }, + ), + ).toMatchObject({ + identity: { state: "ambiguous", candidates: [{ executionId }] }, + ...(nextExecutionCursor ? { nextExecutionCursor } : {}), + }); + } + const firstInspection = inspectExecutionIdentityRun( + { executionId: "execution-first" }, + { ...database, now: 300 }, + ); + const secondInspection = inspectExecutionIdentityRun( + { executionId: "execution-second" }, + { ...database, now: 300 }, + ); + expect(firstInspection).toMatchObject({ + identity: { state: "present", context: { contextId: "context-first" } }, + coverage: { state: "enforced" }, + decisions: [{ decision: { outcome: "not-applicable" } }, { decision: { outcome: "denied" } }], + }); + expect(secondInspection).toMatchObject({ + identity: { state: "present", context: { contextId: "context-second" } }, + coverage: { + state: "unknown", + missingEvidence: expect.arrayContaining(["decision.execution_link"]), + }, + decisions: [ + { decision: { outcome: "not-applicable" } }, + { + decision: { + outcome: "unknown", + reasonCode: "operator_approval_execution_link_mismatch", + }, + }, + ], + }); + }); + + it("confirms durable retries without manufacturing lost evidence", () => { + const database = databaseOptions(); + const envelope = captureExecutionIdentityAdmissionEnvelope(facts("run-recovery"), { + now: 100, + contextId: "context-recovery", + executionId: "execution-recovery", + runtimeInstanceId: "runtime-original", + }); + const original = processExecutionIdentityAdmissionWork( + { kind: "capture", envelope }, + { ...database, now: 100 }, + ); + const token = createExecutionIdentityAdmissionToken("run-recovery", { + now: 100, + contextId: "context-recovery", + executionId: "execution-recovery", + }); + + closeOpenClawStateDatabaseForTest(); + expect( + processExecutionIdentityAdmissionWork({ kind: "retry-reference", token }, database), + ).toEqual(original); + + const missingDatabase = databaseOptions(); + expect(() => + processExecutionIdentityAdmissionWork({ kind: "retry-reference", token }, missingDatabase), + ).toThrow("execution identity recovery evidence unavailable"); + expect( + inspectExecutionIdentityRun({ executionId: "execution-recovery" }, missingDatabase), + ).toMatchObject({ + run: { executionId: "execution-recovery", status: "unknown" }, + identity: { state: "unknown", reasonCode: "execution_not_found" }, + decisions: [], + }); + }); + + it("projects authoritative local CLI and system ingress without conflating them", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-local"), database); + prepareExecutionIdentityContextAtAdmission( + facts("run-system", { + ingress: { kind: "system", boundary: "gateway.boot", state: "present" }, + }), + database, + ); + + expect(inspectExecutionIdentityRun({ runId: "run-local" }, database).identity).toMatchObject({ + state: "present", + context: { + ingress: { kind: "local-cli", boundary: "agent-command.local", state: "present" }, + }, + }); + expect(inspectExecutionIdentityRun({ runId: "run-system" }, database).identity).toMatchObject({ + state: "present", + context: { + ingress: { kind: "system", boundary: "gateway.boot", state: "present" }, + }, + }); + }); + + it("keeps inspection read-only and lets persistence create the additive table", () => { + const database = databaseOptions(); + const reopened = openOpenClawStateDatabase(database); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toBeUndefined(); + expect(inspectExecutionIdentityRun({ runId: "missing" }, database)).toMatchObject({ + run: { status: "unknown" }, + identity: { state: "unknown", reasonCode: "run_not_found" }, + }); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toBeUndefined(); + + prepareExecutionIdentityContextAtAdmission(facts("schema-restored"), database); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toEqual({ name: "execution_identity_contexts" }); + expect( + reopened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'index' AND name = ?") + .get("execution_identity_contexts_run_created_idx"), + ).toEqual({ name: "execution_identity_contexts_run_created_idx" }); + }); + + it("keeps maintenance read-only until the first identity capture", () => { + const database = databaseOptions(); + const opened = openOpenClawStateDatabase(database); + expect( + opened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toBeUndefined(); + + expect(pruneExpiredExecutionIdentityContexts({ database })).toBe(0); + expect( + opened.db + .prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toBeUndefined(); + }); + + it("records attribution only when an invoker fact is actually present", () => { + const database = databaseOptions(); + const context = prepareExecutionIdentityContextAtAdmission( + facts("run-attributed", { + invoker: { + state: "present", + kind: "local-account", + rawPrincipalRef: "private-local-account", + displayLabel: "Operator OPENAI_API_KEY=sk-1234567890abcdef", + }, + applicableGrants: [ + { rawGrantRef: "grant-z", state: "present" }, + { rawGrantRef: "grant-a", state: "present" }, + ], + assurance: [ + { + kind: "local-process", + rawEvidenceRef: "private-process-evidence", + strength: "boundary-verified", + }, + ], + }), + { ...database, runtimeInstanceId: "private-runtime" }, + ); + const encoded = JSON.stringify(context); + + expect(context.coverageState).toBe("attribution-only"); + expect(context.invoker.state).toBe("present"); + expect(context.missingEvidence).toEqual([]); + expect(context.applicableGrants.map((grant) => grant.grantRef)).toEqual( + context.applicableGrants.map((grant) => grant.grantRef).toSorted(), + ); + for (const secret of [ + "private-local-account", + "private-process-evidence", + "private-runtime", + "grant-a", + "grant-z", + "sk-1234567890abcdef", + ]) { + expect(encoded).not.toContain(secret); + } + expect(context.invoker.principal?.principalRef).toMatch(/^hmac-sha256:v1:/u); + }); + + it("declines recording instead of rotating a missing HMAC key with retained contexts", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-before-key-loss"), database); + openOpenClawStateDatabase(database).db.exec("DELETE FROM audit_identity_keys;"); + closeOpenClawStateDatabaseForTest(); + + expect(() => + prepareExecutionIdentityContextAtAdmission(facts("run-after-key-loss"), database), + ).toThrow("audit identity key is missing"); + }); + + it("skips new context rows when audit collection is disabled", () => { + const database = databaseOptions(); + + expect( + enqueueExecutionIdentityContextAtAdmission(facts("run-disabled"), { enabled: false }), + ).toBeUndefined(); + + expect( + openOpenClawStateDatabase(database) + .db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get("execution_identity_contexts"), + ).toBeUndefined(); + }); + + it("keeps bounded retention maintenance available while collection is disabled", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-before-disable"), { + ...database, + now: 0, + runtimeInstanceId: "runtime-1", + }); + expect( + enqueueExecutionIdentityContextAtAdmission(facts("run-disabled"), { enabled: false }), + ).toBeUndefined(); + + expect(pruneExpiredExecutionIdentityContexts({ database, now: RETENTION_MS + 1 })).toBe(1); + expect( + openOpenClawStateDatabase(database) + .db.prepare("SELECT COUNT(*) AS count FROM execution_identity_contexts") + .get(), + ).toEqual({ count: 0 }); + }); + + it("leaves the original context intact when a later worker write conflicts", () => { + const database = databaseOptions(); + const envelope = captureExecutionIdentityAdmissionEnvelope(facts("run-best-effort"), { + contextId: "context-best-effort", + executionId: "execution-best-effort", + runtimeInstanceId: "runtime-1", + }); + persistExecutionIdentityAdmissionEnvelope(envelope, database); + + expect(() => + persistExecutionIdentityAdmissionEnvelope( + { ...envelope, agentId: "conflicting-agent" }, + database, + ), + ).toThrow("execution identity context conflict"); + }); + + it("rolls back the worker write when insert-time retention cleanup fails", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-expired"), { + ...database, + now: 0, + runtimeInstanceId: "runtime-1", + }); + openOpenClawStateDatabase(database).db.exec(` + CREATE TRIGGER reject_identity_cleanup + BEFORE DELETE ON execution_identity_contexts + BEGIN + SELECT RAISE(ABORT, 'cleanup unavailable'); + END; + `); + + expect(() => + prepareExecutionIdentityContextAtAdmission(facts("run-still-admitted"), { + ...database, + now: RETENTION_MS + 1, + runtimeInstanceId: "runtime-1", + }), + ).toThrow("cleanup unavailable"); + }); + + it("stops projecting context and decisions immediately after the retention boundary", () => { + const database = databaseOptions(); + const createdAt = 1_000; + prepareExecutionIdentityContextAtAdmission(facts("run-retention"), { + ...database, + now: createdAt, + contextId: "expired-context-secret", + executionId: "expired-execution-secret", + runtimeInstanceId: "expired-runtime-secret", + }); + + const immediatelyBefore = inspectExecutionIdentityRun( + { runId: "run-retention" }, + { ...database, now: createdAt + RETENTION_MS - 1 }, + ); + expect(immediatelyBefore.identity).toMatchObject({ + state: "present", + context: { contextId: "expired-context-secret" }, + }); + expect(immediatelyBefore.decisions).toHaveLength(1); + expect( + inspectExecutionIdentityRun( + { runId: "run-retention" }, + { ...database, now: createdAt + RETENTION_MS }, + ).identity.state, + ).toBe("present"); + + const immediatelyAfter = inspectExecutionIdentityRun( + { runId: "run-retention" }, + { ...database, now: createdAt + RETENTION_MS + 1 }, + ); + expect(immediatelyAfter).toMatchObject({ + run: { status: "known" }, + identity: { + state: "unsupported", + reasonCode: "identity_context_unavailable", + remediation: [ + expect.objectContaining({ + code: "run_again_after_expiry", + text: expect.stringContaining("outside the 30-day window"), + }), + ], + }, + decisions: [], + coverage: { state: "unsupported", missingEvidence: ["identity.context"] }, + }); + expect(JSON.stringify(immediatelyAfter)).not.toContain("expired-context-secret"); + expect(JSON.stringify(immediatelyAfter)).not.toContain("expired-runtime-secret"); + expect(JSON.stringify(immediatelyAfter)).not.toContain("run_admission_identity_not_evaluated"); + const exactAfter = inspectExecutionIdentityRun( + { executionId: "expired-execution-secret" }, + { ...database, now: createdAt + RETENTION_MS + 1 }, + ); + expect(exactAfter).toMatchObject({ + run: { executionId: "expired-execution-secret", status: "known" }, + identity: { state: "unsupported", reasonCode: "identity_context_unavailable" }, + decisions: [], + }); + expect(JSON.stringify(exactAfter)).not.toContain("expired-context-secret"); + expect(JSON.stringify(exactAfter)).not.toContain("expired-runtime-secret"); + + closeOpenClawStateDatabaseForTest(); + expect( + inspectExecutionIdentityRun( + { runId: "run-retention" }, + { ...database, now: createdAt + RETENTION_MS + 1 }, + ), + ).toEqual(immediatelyAfter); + + expect( + pruneExpiredExecutionIdentityContexts({ + database, + now: createdAt + RETENTION_MS + 1, + }), + ).toBe(1); + expect( + inspectExecutionIdentityRun( + { runId: "run-retention" }, + { ...database, now: createdAt + RETENTION_MS + 1 }, + ), + ).toMatchObject({ + run: { status: "unknown" }, + identity: { + state: "unknown", + reasonCode: "run_not_found", + remediation: [ + expect.objectContaining({ text: expect.stringContaining("not proof of no run") }), + ], + }, + decisions: [], + }); + }); + + it("prunes expired contexts in bounded maintenance batches without new inserts", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("schema-seed"), { + ...database, + now: 1, + runtimeInstanceId: "runtime-1", + }); + const { db } = openOpenClawStateDatabase(database); + db.exec("DELETE FROM execution_identity_contexts;"); + db.prepare( + `WITH RECURSIVE rows(n) AS ( + VALUES (1) + UNION ALL + SELECT n + 1 FROM rows WHERE n < 1025 + ) + INSERT INTO execution_identity_contexts ( + context_id, execution_id, run_id, created_at, coverage_state, context_bytes, context_json + ) + SELECT 'context-' || n, 'execution-' || n, 'run-' || n, 0, 'unattributed', 2, '{}' + FROM rows`, + ).run(); + + expect(pruneExpiredExecutionIdentityContexts({ database, now: RETENTION_MS + 1 })).toBe(1_024); + expect(db.prepare("SELECT COUNT(*) AS count FROM execution_identity_contexts").get()).toEqual({ + count: 1, + }); + expect(pruneExpiredExecutionIdentityContexts({ database, now: RETENTION_MS + 1 })).toBe(1); + expect(db.prepare("SELECT COUNT(*) AS count FROM execution_identity_contexts").get()).toEqual({ + count: 0, + }); + }); + + it("prunes retention and row-cap overflow in bounded batches", () => { + const retentionDatabase = databaseOptions(); + for (const runId of ["old-1", "old-2", "old-3"]) { + prepareExecutionIdentityContextAtAdmission(facts(runId), { + ...retentionDatabase, + now: 0, + runtimeInstanceId: "runtime-1", + limits: { maxRows: 10, pruneBatchRows: 1 }, + }); + } + prepareExecutionIdentityContextAtAdmission(facts("new-1"), { + ...retentionDatabase, + now: RETENTION_MS + 1, + runtimeInstanceId: "runtime-1", + limits: { maxRows: 1, pruneBatchRows: 1 }, + }); + const retainedAfterOneBatch = openOpenClawStateDatabase(retentionDatabase) + .db.prepare("SELECT COUNT(*) AS count FROM execution_identity_contexts") + .get() as { count: number }; + expect(retainedAfterOneBatch.count).toBe(3); + + const capDatabase = databaseOptions(); + for (const runId of ["cap-1", "cap-2", "cap-3"]) { + prepareExecutionIdentityContextAtAdmission(facts(runId), { + ...capDatabase, + now: 100, + contextId: `context-${runId}`, + runtimeInstanceId: "runtime-1", + limits: { maxRows: 2, pruneBatchRows: 1 }, + }); + } + const capped = openOpenClawStateDatabase(capDatabase) + .db.prepare("SELECT run_id FROM execution_identity_contexts ORDER BY context_id") + .all() as Array<{ run_id: string }>; + expect(capped).toHaveLength(2); + expect(capped.map((row) => row.run_id)).not.toContain("cap-1"); + }); + + it("enforces the row cap across independent database connections", () => { + const database = databaseOptions(); + const path = openOpenClawStateDatabase(database).path; + closeOpenClawStateDatabaseForTest(); + const first = openIndependentStateDatabase(path); + const second = openIndependentStateDatabase(path); + try { + for (const [index, connection] of [first, second, first, second].entries()) { + const runId = `shared-cap-${index + 1}`; + prepareExecutionIdentityContextAtAdmission(facts(runId), { + database: connection, + now: 100 + index, + contextId: `context-${runId}`, + runtimeInstanceId: "runtime-1", + limits: { maxRows: 2, pruneBatchRows: 1 }, + }); + } + + const retained = first.db + .prepare("SELECT run_id FROM execution_identity_contexts ORDER BY created_at") + .all() as Array<{ run_id: string }>; + expect(retained.map((row) => row.run_id)).toEqual(["shared-cap-3", "shared-cap-4"]); + } finally { + first.db.close(); + second.db.close(); + } + }); + + it("inspects through a read-only connection while another writer holds the database", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("held-lock-inspection"), { + ...database, + now: 100, + contextId: "context-held-lock-inspection", + executionId: "execution-held-lock-inspection", + runtimeInstanceId: "runtime-1", + }); + const path = openOpenClawStateDatabase(database).path; + closeOpenClawStateDatabaseForTest(); + const lockDatabase = openNodeSqliteDatabase(path); + lockDatabase.exec("BEGIN IMMEDIATE"); + try { + expect( + inspectExecutionIdentityRun( + { executionId: "execution-held-lock-inspection" }, + { ...database, now: 100 }, + ), + ).toMatchObject({ + identity: { + state: "present", + context: { + contextId: "context-held-lock-inspection", + executionId: "execution-held-lock-inspection", + runId: "held-lock-inspection", + }, + }, + }); + } finally { + lockDatabase.exec("ROLLBACK"); + lockDatabase.close(); + } + }); + + it("returns typed corrupt, unknown, and unsupported projections", () => { + const corruptDatabase = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-corrupt"), { + ...corruptDatabase, + runtimeInstanceId: "runtime-1", + }); + openOpenClawStateDatabase(corruptDatabase) + .db.prepare("UPDATE execution_identity_contexts SET context_json = ? WHERE run_id = ?") + .run("{", "run-corrupt"); + expect(inspectExecutionIdentityRun({ runId: "run-corrupt" }, corruptDatabase)).toMatchObject({ + run: { status: "known" }, + identity: { state: "unknown", reasonCode: "identity_context_corrupt" }, + coverage: { state: "unknown" }, + }); + + const unknownDatabase = databaseOptions(); + expect(inspectExecutionIdentityRun({ runId: "never-seen" }, unknownDatabase)).toMatchObject({ + run: { status: "unknown" }, + identity: { + state: "unknown", + reasonCode: "run_not_found", + remediation: [expect.objectContaining({ code: "verify_run_id" })], + }, + }); + + recordAuditEvent( + { + sourceId: "legacy-run:1", + sourceSequence: 1, + occurredAt: Date.now(), + kind: "agent_run", + action: "agent.run.started", + status: "started", + actorType: "agent", + actorId: "main", + agentId: "main", + runId: "legacy-run", + }, + unknownDatabase, + ); + expect(inspectExecutionIdentityRun({ runId: "legacy-run" }, unknownDatabase)).toMatchObject({ + run: { status: "known" }, + identity: { + state: "unsupported", + reasonCode: "identity_context_unavailable", + remediation: [expect.objectContaining({ code: "record_new_identity_context" })], + }, + }); + }); + + it("projects one non-enforcement admission explanation", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-receipt"), { + ...database, + now: 123, + contextId: "context-receipt", + runtimeInstanceId: "runtime-1", + }); + const result = inspectExecutionIdentityRun({ runId: "run-receipt" }, { ...database, now: 123 }); + + expect(result.identity).toMatchObject({ + state: "present", + context: { contextId: "context-receipt", coverageState: "unattributed" }, + }); + expect(result.decisions).toEqual([ + expect.objectContaining({ + decision: { + outcome: "not-applicable", + reasonCode: "run_admission_identity_not_evaluated", + }, + enforcement: expect.objectContaining({ + coverageState: "unattributed", + policyRefs: [], + grantRefs: [], + contextFieldsUsed: [], + }), + }), + ]); + expect( + inspectExecutionIdentityRun( + { runId: "run-receipt", decisionLimit: 1 }, + { ...database, now: 123 }, + ).nextDecisionCursor, + ).toBeUndefined(); + expect( + inspectExecutionIdentityRun( + { runId: "run-receipt", decisionCursor: "a:0:0" }, + { ...database, now: 123 }, + ).decisions, + ).toEqual([]); + }); + + it("projects an authoritative denied approval by run before and after restart", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-denied-receipt"), { + ...database, + now: 100, + contextId: "context-denied-receipt", + executionId: "execution-denied-receipt", + runtimeInstanceId: "runtime-1", + }); + recordDeniedApprovalForRun("run-denied-receipt", database, "denied-approval", { + contextId: "context-denied-receipt", + executionId: "execution-denied-receipt", + }); + + const beforeRestart = inspectExecutionIdentityRun( + { runId: "run-denied-receipt" }, + { ...database, now: 300 }, + ); + expect(beforeRestart).toMatchObject({ + coverage: { state: "enforced" }, + decisions: [ + { decision: { outcome: "not-applicable" } }, + { + contextId: "context-denied-receipt", + executionId: "execution-denied-receipt", + runId: "run-denied-receipt", + decision: { + outcome: "denied", + reasonCode: "operator_approval_denied_by_reviewer", + }, + enforcement: { + coverageState: "enforced", + contextFieldsUsed: ["contextId", "executionId", "runId"], + }, + source: { owner: "operator_approvals" }, + }, + ], + }); + expect(JSON.stringify(beforeRestart)).not.toContain("private-reviewer-device"); + expect(JSON.stringify(beforeRestart)).not.toContain("private-tool-call"); + + closeOpenClawStateDatabaseForTest(); + expect( + inspectExecutionIdentityRun({ runId: "run-denied-receipt" }, { ...database, now: 300 }), + ).toEqual(beforeRestart); + expect( + inspectExecutionIdentityRun( + { runId: "run-denied-receipt", decisionCursor: "a:0:0", decisionLimit: 1 }, + { ...database, now: 300 }, + ), + ).toMatchObject({ + decisions: [{ decision: { reasonCode: "operator_approval_denied_by_reviewer" } }], + }); + expect( + inspectExecutionIdentityRun( + { runId: "run-denied-receipt", decisionCursor: "1", decisionLimit: 1 }, + { ...database, now: 300 }, + ).decisions, + ).toMatchObject([{ decision: { reasonCode: "operator_approval_denied_by_reviewer" } }]); + }); + + it("keeps a corrupt approval unknown before its decision page is returned", () => { + const database = databaseOptions(); + prepareExecutionIdentityContextAtAdmission(facts("run-corrupt-approval"), { + ...database, + now: 100, + contextId: "context-corrupt-approval", + executionId: "execution-corrupt-approval", + runtimeInstanceId: "runtime-1", + }); + recordDeniedApprovalForRun("run-corrupt-approval", database, "corrupt-approval", { + contextId: "context-corrupt-approval", + executionId: "execution-corrupt-approval", + }); + openOpenClawStateDatabase(database) + .db.prepare("UPDATE operator_approvals SET presentation_json = ? WHERE approval_id = ?") + .run("{", "corrupt-approval"); + + expect( + inspectExecutionIdentityRun( + { executionId: "execution-corrupt-approval", decisionLimit: 1 }, + { ...database, now: 300 }, + ), + ).toMatchObject({ + coverage: { + state: "unknown", + missingEvidence: expect.arrayContaining(["operator_approval.valid"]), + }, + decisions: [{ decision: { outcome: "not-applicable" } }], + nextDecisionCursor: "a:0:0", + }); + }); + + it("reports a retained approval with no identity context as an unknown missing link", () => { + const database = databaseOptions(); + recordDeniedApprovalForRun("run-missing-context", database); + + expect( + inspectExecutionIdentityRun({ runId: "run-missing-context" }, { ...database, now: 300 }), + ).toMatchObject({ + run: { runId: "run-missing-context", status: "known" }, + identity: { + state: "unknown", + reasonCode: "decision_context_link_missing", + }, + decisions: [], + coverage: { + state: "unknown", + missingEvidence: ["identity.context", "decision.context_link"], + }, + }); + }); +}); diff --git a/src/audit/execution-owner-lifecycle-binding-store.ts b/src/audit/execution-owner-lifecycle-binding-store.ts new file mode 100644 index 0000000000000000000000000000000000000000..4eddebec5fdc4205c0a3ed6030c8fb260d5444d4 --- /dev/null +++ b/src/audit/execution-owner-lifecycle-binding-store.ts @@ -0,0 +1,121 @@ +import type { DatabaseSync } from "node:sqlite"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js"; +import { + classifyExecutionOwnerBinding, + type ExecutionOwnerBindingResult, +} from "./execution-owner-binding.js"; + +export const EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE = + "execution_owner_lifecycle_bindings" as const; +type ExecutionOwnerLifecycleKind = "cron" | "task" | "flow"; + +type ExecutionOwnerLifecycleDatabase = Pick< + OpenClawStateDatabase, + "execution_owner_lifecycle_bindings" +>; + +const SCHEMA_START = `CREATE TABLE IF NOT EXISTS ${EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE} (`; +const SCHEMA_END = ") STRICT;"; + +function lifecycleDb(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +/** Creates only the canonical additive metadata table at first admitted owner use. */ +function ensureExecutionOwnerLifecycleBindingSchema(db: DatabaseSync): void { + if (tableExists(db, EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE)) { + return; + } + const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(SCHEMA_START); + const end = start < 0 ? -1 : OPENCLAW_STATE_SCHEMA_SQL.indexOf(SCHEMA_END, start); + if (start < 0 || end < start) { + throw new Error("OpenClaw execution owner lifecycle binding schema marker is missing."); + } + // sqlite-allow-raw -- Canonical feature-local additive DDL only; metadata rows use Kysely. + db.exec(OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + SCHEMA_END.length)); +} + +function classifyRetainedBinding( + current: { context_id: string; execution_id: string }, + binding: { contextId: string; executionId: string }, +): "already-bound" | "mismatch" { + const state = classifyExecutionOwnerBinding( + { contextId: current.context_id, executionId: current.execution_id }, + binding, + ); + return state === "unbound" ? "mismatch" : state; +} + +/** Stores one exact admission identity after its canonical owner row has been revalidated. */ +export function bindExecutionOwnerLifecycleMetadata(params: { + db: DatabaseSync; + ownerKind: ExecutionOwnerLifecycleKind; + ownerId: string; + binding: { contextId: string; executionId: string }; +}): Exclude { + ensureExecutionOwnerLifecycleBindingSchema(params.db); + const database = lifecycleDb(params.db); + const current = executeSqliteQueryTakeFirstSync( + params.db, + database + .selectFrom(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + .select(["context_id", "execution_id"]) + .where("owner_kind", "=", params.ownerKind) + .where("owner_id", "=", params.ownerId), + ); + if (current) { + return classifyRetainedBinding(current, params.binding); + } + const inserted = executeSqliteQuerySync( + params.db, + database + .insertInto(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + .values({ + owner_kind: params.ownerKind, + owner_id: params.ownerId, + context_id: params.binding.contextId, + execution_id: params.binding.executionId, + }) + .onConflict((conflict) => conflict.columns(["owner_kind", "owner_id"]).doNothing()), + ); + if (Number(inserted.numAffectedRows ?? 0n) === 1) { + return "bound"; + } + const raced = executeSqliteQueryTakeFirstSync( + params.db, + database + .selectFrom(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + .select(["context_id", "execution_id"]) + .where("owner_kind", "=", params.ownerKind) + .where("owner_id", "=", params.ownerId), + ); + return raced ? classifyRetainedBinding(raced, params.binding) : "mismatch"; +} + +/** Removes exact owner metadata without allocating the opt-in table. */ +export function deleteExecutionOwnerLifecycleMetadata(params: { + db: DatabaseSync; + ownerKind: ExecutionOwnerLifecycleKind; + ownerIds: readonly string[]; +}): void { + if ( + params.ownerIds.length === 0 || + !tableExists(params.db, EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + ) { + return; + } + executeSqliteQuerySync( + params.db, + lifecycleDb(params.db) + .deleteFrom(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + .where("owner_kind", "=", params.ownerKind) + .where("owner_id", "in", params.ownerIds), + ); +} diff --git a/src/audit/execution-owner-lifecycle-receipts.ts b/src/audit/execution-owner-lifecycle-receipts.ts new file mode 100644 index 0000000000000000000000000000000000000000..442598a068a8ddc7d7308cb1a12132429c37022b --- /dev/null +++ b/src/audit/execution-owner-lifecycle-receipts.ts @@ -0,0 +1,445 @@ +/** Static owner-native cron/task/flow lifecycle projection for run inspection. */ +import type { DatabaseSync } from "node:sqlite"; +import type { + DecisionReceiptV1, + ExecutionIdentityContextV1, +} from "../../packages/gateway-protocol/src/index.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { runSqliteDeferredTransactionSync } from "../infra/sqlite-transaction.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; +import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE } from "./execution-owner-lifecycle-binding-store.js"; + +type WithSqliteRowId = Row & { rowid: number }; +type OwnerLifecycleDatabase = { + cron_run_receipts: WithSqliteRowId; + execution_owner_lifecycle_bindings: OpenClawStateDatabase["execution_owner_lifecycle_bindings"]; + flow_runs: WithSqliteRowId; + task_runs: WithSqliteRowId; +}; +export type OwnerLifecycleStage = "cron" | "task" | "flow"; +export type OwnerLifecycleCursor = { occurredAt: number; rowId: number }; +type OwnerLifecycleDisplayProducer = "cron-lifecycle" | "task-lifecycle" | "flow-lifecycle"; +type OwnerLifecycleReceiptEntry = { + receipt: DecisionReceiptV1; + selectorId: string; + displayProducer: OwnerLifecycleDisplayProducer; +}; +type OwnerLifecycleRow = { + executionId: string | null; + occurredAt: number; + owner: "cron_run_receipts" | "task_runs" | "flow_runs"; + recordId: string; + rowId: number; + status: string; +}; + +const KNOWN_STATUSES: Record> = { + cron: new Set(["running", "ok", "error", "skipped", "interrupted", "superseded"]), + task: new Set([ + "queued", + "running", + "succeeded", + "failed", + "timed_out", + "cancelled", + "lost", + "blocked", + ]), + flow: new Set([ + "queued", + "running", + "waiting", + "blocked", + "succeeded", + "failed", + "cancelled", + "lost", + ]), +}; +const OWNER_LIFECYCLE_CURSOR_RETAINED_ERROR = "owner lifecycle cursor is no longer retained"; + +function ownerName(stage: OwnerLifecycleStage): OwnerLifecycleRow["owner"] { + return stage === "cron" ? "cron_run_receipts" : stage === "task" ? "task_runs" : "flow_runs"; +} + +function displayProducer(stage: OwnerLifecycleStage): OwnerLifecycleDisplayProducer { + return stage === "cron" + ? "cron-lifecycle" + : stage === "task" + ? "task-lifecycle" + : "flow-lifecycle"; +} + +function assertRetainedCursor(params: { + db: DatabaseSync; + stage: OwnerLifecycleStage; + contextId: string; + executionId: string; + after?: OwnerLifecycleCursor; +}): void { + if (!params.after) { + return; + } + const kysely = getNodeSqliteKysely(params.db); + const ownerQuery = + params.stage === "cron" + ? kysely + .selectFrom("cron_run_receipts") + .select("receipt_id as ownerId") + .where("rowid", "=", params.after.rowId) + .where("started_at_ms", "=", params.after.occurredAt) + : params.stage === "task" + ? kysely + .selectFrom("task_runs") + .select("task_id as ownerId") + .where("rowid", "=", params.after.rowId) + .where("created_at", "=", params.after.occurredAt) + : kysely + .selectFrom("flow_runs") + .select("flow_id as ownerId") + .where("rowid", "=", params.after.rowId) + .where("created_at", "=", params.after.occurredAt); + const owner = executeSqliteQueryTakeFirstSync(params.db, ownerQuery); + // Admission binds at most one owner per lifecycle kind to an execution. + // The exact execution match therefore rejects a different owner reusing this rowid. + const binding = owner + ? executeSqliteQueryTakeFirstSync( + params.db, + kysely + .selectFrom(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + .select("owner_id") + .where("owner_kind", "=", params.stage) + .where("owner_id", "=", owner.ownerId) + .where("context_id", "=", params.contextId) + .where("execution_id", "=", params.executionId), + ) + : undefined; + if (!binding) { + throw new Error(OWNER_LIFECYCLE_CURSOR_RETAINED_ERROR); + } +} + +function readRows(params: { + db: DatabaseSync; + stage: OwnerLifecycleStage; + contextId: string; + executionId: string; + after?: OwnerLifecycleCursor; + offset?: number; + limit: number; +}): OwnerLifecycleRow[] { + const owner = ownerName(params.stage); + if ( + !tableExists(params.db, owner) || + !tableExists(params.db, EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + ) { + if (params.after) { + throw new Error(OWNER_LIFECYCLE_CURSOR_RETAINED_ERROR); + } + return []; + } + assertRetainedCursor(params); + const kysely = getNodeSqliteKysely(params.db); + if (params.stage === "cron") { + let query = kysely + .selectFrom("cron_run_receipts") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef("execution_owner_lifecycle_bindings.owner_id", "=", "cron_run_receipts.receipt_id") + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "cron"), + ) + .select([ + "cron_run_receipts.receipt_id as recordId", + "execution_owner_lifecycle_bindings.execution_id as executionId", + "cron_run_receipts.started_at_ms as occurredAt", + "cron_run_receipts.status", + "cron_run_receipts.rowid as rowId", + ]) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .orderBy("cron_run_receipts.started_at_ms", "asc") + .orderBy("cron_run_receipts.rowid", "asc") + .limit(params.limit); + if (params.after) { + query = query.where((eb) => + eb.or([ + eb("cron_run_receipts.started_at_ms", ">", params.after!.occurredAt), + eb.and([ + eb("cron_run_receipts.started_at_ms", "=", params.after!.occurredAt), + eb("cron_run_receipts.rowid", ">", params.after!.rowId), + ]), + ]), + ); + } else if (params.offset) { + query = query.offset(params.offset); + } + return executeSqliteQuerySync(params.db, query).rows.map((row) => + Object.assign(row, { owner }), + ); + } + if (params.stage === "task") { + let query = kysely + .selectFrom("task_runs") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef("execution_owner_lifecycle_bindings.owner_id", "=", "task_runs.task_id") + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "task"), + ) + .select([ + "task_runs.task_id as recordId", + "execution_owner_lifecycle_bindings.execution_id as executionId", + "task_runs.created_at as occurredAt", + "task_runs.status", + "task_runs.terminal_outcome as terminalOutcome", + "task_runs.rowid as rowId", + ]) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .orderBy("task_runs.created_at", "asc") + .orderBy("task_runs.rowid", "asc") + .limit(params.limit); + if (params.after) { + query = query.where((eb) => + eb.or([ + eb("task_runs.created_at", ">", params.after!.occurredAt), + eb.and([ + eb("task_runs.created_at", "=", params.after!.occurredAt), + eb("task_runs.rowid", ">", params.after!.rowId), + ]), + ]), + ); + } else if (params.offset) { + query = query.offset(params.offset); + } + return executeSqliteQuerySync(params.db, query).rows.map((row) => + Object.assign(row, { + owner, + status: row.terminalOutcome === "blocked" ? "blocked" : row.status, + }), + ); + } + let query = kysely + .selectFrom("flow_runs") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef("execution_owner_lifecycle_bindings.owner_id", "=", "flow_runs.flow_id") + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "flow"), + ) + .select([ + "flow_runs.flow_id as recordId", + "execution_owner_lifecycle_bindings.execution_id as executionId", + "flow_runs.created_at as occurredAt", + "flow_runs.status", + "flow_runs.rowid as rowId", + ]) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .orderBy("flow_runs.created_at", "asc") + .orderBy("flow_runs.rowid", "asc") + .limit(params.limit); + if (params.after) { + query = query.where((eb) => + eb.or([ + eb("flow_runs.created_at", ">", params.after!.occurredAt), + eb.and([ + eb("flow_runs.created_at", "=", params.after!.occurredAt), + eb("flow_runs.rowid", ">", params.after!.rowId), + ]), + ]), + ); + } else if (params.offset) { + query = query.offset(params.offset); + } + return executeSqliteQuerySync(params.db, query).rows.map((row) => Object.assign(row, { owner })); +} + +function countRows(params: { + db: DatabaseSync; + stage: OwnerLifecycleStage; + contextId: string; + executionId?: string; +}): number { + const owner = ownerName(params.stage); + if ( + !tableExists(params.db, owner) || + !tableExists(params.db, EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE) + ) { + return 0; + } + const kysely = getNodeSqliteKysely(params.db); + const query = + params.stage === "cron" + ? kysely + .selectFrom("cron_run_receipts") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef( + "execution_owner_lifecycle_bindings.owner_id", + "=", + "cron_run_receipts.receipt_id", + ) + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "cron"), + ) + .select((eb) => eb.fn.countAll().as("count")) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .$if(params.executionId !== undefined, (qb) => + qb.where("execution_owner_lifecycle_bindings.execution_id", "=", params.executionId!), + ) + : params.stage === "task" + ? kysely + .selectFrom("task_runs") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef("execution_owner_lifecycle_bindings.owner_id", "=", "task_runs.task_id") + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "task"), + ) + .select((eb) => eb.fn.countAll().as("count")) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .$if(params.executionId !== undefined, (qb) => + qb.where("execution_owner_lifecycle_bindings.execution_id", "=", params.executionId!), + ) + : kysely + .selectFrom("flow_runs") + .innerJoin(EXECUTION_OWNER_LIFECYCLE_BINDING_TABLE, (join) => + join + .onRef("execution_owner_lifecycle_bindings.owner_id", "=", "flow_runs.flow_id") + .on("execution_owner_lifecycle_bindings.owner_kind", "=", "flow"), + ) + .select((eb) => eb.fn.countAll().as("count")) + .where("execution_owner_lifecycle_bindings.context_id", "=", params.contextId) + .$if(params.executionId !== undefined, (qb) => + qb.where("execution_owner_lifecycle_bindings.execution_id", "=", params.executionId!), + ); + return executeSqliteQueryTakeFirstSync(params.db, query)?.count ?? 0; +} + +function projectReceipt( + stage: OwnerLifecycleStage, + row: OwnerLifecycleRow, + context: ExecutionIdentityContextV1, +): DecisionReceiptV1 { + const exact = row.executionId === context.executionId; + const known = KNOWN_STATUSES[stage].has(row.status); + const valid = exact && known; + const missingEvidence = valid + ? [] + : [exact ? `decision.${stage}_owner_status` : "decision.execution_link"]; + return { + schemaVersion: 1, + receiptId: `${stage}:${row.recordId}`, + contextId: context.contextId, + executionId: context.executionId, + runId: context.runId, + actionId: row.recordId, + occurredAt: row.occurredAt, + action: { + family: stage === "cron" ? "scheduled-run" : stage === "task" ? "task" : "flow", + operation: "lifecycle", + summary: valid + ? `${stage === "cron" ? "Scheduled run" : stage === "task" ? "Task" : "Flow"} lifecycle: ${row.status.replaceAll("_", "-")}.` + : "Owner lifecycle evidence could not be matched exactly.", + }, + decision: { + outcome: valid ? "not-applicable" : "unknown", + reasonCode: valid + ? `${stage}_run_${row.status}` + : exact + ? `${stage}_run_status_unknown` + : `${stage}_run_execution_link_mismatch`, + }, + enforcement: { + coverageState: valid ? "attribution-only" : "unknown", + evaluatorRef: `${stage}-lifecycle-owner`, + policyRefs: [], + grantRefs: [], + contextFieldsUsed: ["contextId", "executionId"], + }, + source: { + owner: row.owner, + recordRef: row.recordId, + decisionBoundary: `${stage}.run.lifecycle`, + }, + missingEvidence, + remediation: valid + ? [] + : [ + { + code: "inspect_owner_execution_binding", + text: "Inspect the owner row and its exact admission binding before drawing a lifecycle conclusion.", + }, + ], + }; +} + +export function summarizeOwnerLifecycleReceipts(params: { + stage: OwnerLifecycleStage; + context: ExecutionIdentityContextV1; + options: OpenClawStateDatabaseOptions; +}): { count: number; coverageState?: "attribution-only" | "unknown"; missingEvidence: string[] } { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const count = countRows({ db, stage: params.stage, contextId: params.context.contextId }); + const exactCount = countRows({ + db, + stage: params.stage, + contextId: params.context.contextId, + executionId: params.context.executionId, + }); + const mismatch = count !== exactCount; + return { + count, + ...(count > 0 + ? { coverageState: mismatch ? ("unknown" as const) : ("attribution-only" as const) } + : {}), + missingEvidence: mismatch ? ["decision.execution_link"] : [], + }; + }, params.options) ?? { count: 0, missingEvidence: [] } + ); +} + +export function pageOwnerLifecycleReceipts(params: { + stage: OwnerLifecycleStage; + context: ExecutionIdentityContextV1; + after?: OwnerLifecycleCursor; + offset?: number; + limit: number; + options: OpenClawStateDatabaseOptions; +}): { entries: OwnerLifecycleReceiptEntry[]; nextCursor?: OwnerLifecycleCursor } { + const retainedRows = withExistingOpenClawStateDatabaseReadOnly( + ({ db }) => + runSqliteDeferredTransactionSync( + db, + () => + readRows({ + db, + stage: params.stage, + contextId: params.context.contextId, + executionId: params.context.executionId, + after: params.after, + offset: params.offset, + limit: params.limit + 1, + }), + { operationLabel: "owner lifecycle receipt page" }, + ), + params.options, + ); + if (!retainedRows && params.after) { + throw new Error(OWNER_LIFECYCLE_CURSOR_RETAINED_ERROR); + } + const rows = retainedRows ?? []; + const hasMore = rows.length > params.limit; + const page = hasMore ? rows.slice(0, params.limit) : rows; + const last = page.at(-1); + return { + entries: page.map((row) => ({ + receipt: projectReceipt(params.stage, row, params.context), + selectorId: `${params.stage}-lifecycle:${row.occurredAt}:${row.rowId}`, + displayProducer: displayProducer(params.stage), + })), + ...(hasMore && last ? { nextCursor: { occurredAt: last.occurredAt, rowId: last.rowId } } : {}), + }; +} diff --git a/src/audit/message-audit-events.ts b/src/audit/message-audit-events.ts new file mode 100644 index 0000000000000000000000000000000000000000..90d5964dd0c79d7046cc755fba6cfa495e2eb283 --- /dev/null +++ b/src/audit/message-audit-events.ts @@ -0,0 +1,37 @@ +import { resolveGlobalSet } from "../shared/global-singleton.js"; +/** Trusted in-process message lifecycle stream for durable audit projection. */ +import { notifyListeners, registerListener } from "../shared/listeners.js"; +import type { MessageAuditEventInput } from "./audit-event-types.js"; + +type TrustedMessageAuditEventVariant = Event extends MessageAuditEventInput + ? Omit & { + /** Optional stable producer identity for replay-safe durable projections. */ + sourceId?: string; + } + : never; + +export type TrustedMessageAuditEvent = TrustedMessageAuditEventVariant; + +type MessageAuditListener = (event: TrustedMessageAuditEvent) => void; + +const listeners = resolveGlobalSet( + Symbol.for("openclaw.messageAuditListeners"), + "close-and-restart", +); + +/** Emit only closed metadata. This stream is intentionally not part of the plugin SDK. */ +export function emitTrustedMessageAuditEvent(event: TrustedMessageAuditEvent): void { + if (listeners.size === 0) { + return; + } + notifyListeners(listeners, event); +} + +export function onTrustedMessageAuditEvent(listener: MessageAuditListener): () => void { + return registerListener(listeners, listener); +} + +/** Lets hot producers skip attribution work while message audit is disabled. */ +export function hasTrustedMessageAuditListeners(): boolean { + return listeners.size > 0; +} diff --git a/src/audit/message-delivery-audit-store.ts b/src/audit/message-delivery-audit-store.ts new file mode 100644 index 0000000000000000000000000000000000000000..2676a1f7c2eb6620114b16eb28dc00ceccd2860b --- /dev/null +++ b/src/audit/message-delivery-audit-store.ts @@ -0,0 +1,347 @@ +/** Read-only owner-native outbound message lifecycle queries for run inspection. */ +import type { DatabaseSync } from "node:sqlite"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { normalizeSqliteNumber } from "../infra/sqlite-number.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; +import { tableExists } from "../state/openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { AUDIT_EVENT_RETENTION_MS, rowToAuditEvent } from "./audit-event-store.js"; +import type { OutboundMessageAuditEventRecord } from "./audit-event-types.js"; +import { + countOutboundMessageProgressForRun, + hasOutboundMessageProgressCursor, + readOutboundMessageProgressForRun, +} from "./message-delivery-progress-store.js"; +import { selectMessageExecutionBinding } from "./message-execution-binding.js"; + +type MessageDeliveryAuditDatabase = Pick< + OpenClawStateKyselyDatabase, + "audit_events" | "outbound_message_execution_bindings" +>; + +function deliveryAuditDb(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +export type OutboundMessageAuditEventCursor = { occurredAt: number; rowId: number }; + +type OwnedMessageEvent = { + event: OutboundMessageAuditEventRecord; + rowId: number; +}; + +type MessageExecutionSelector = { + runId: string; + contextId?: string; + executionId?: string; +}; + +function terminalBindingEventIds( + db: DatabaseSync, + runId: string, + binding: { contextId: string; executionId: string }, +) { + return deliveryAuditDb(db) + .selectFrom("outbound_message_execution_bindings") + .select("event_id") + .where("context_id", "=", binding.contextId) + .where("execution_id", "=", binding.executionId) + .where("run_id", "=", runId); +} + +const MESSAGE_CURSOR_STAGE_SPAN = 1_000_000_000_000; +const MESSAGE_STREAM_CHUNK_SIZE = 256; +type MessageStage = 0 | 1 | 2; +type MessageStream = { + stage: MessageStage; + after?: { occurredAt: number; sequence: number }; + buffered: OwnedMessageEvent[]; + exhausted: boolean; +}; + +function messageStage(event: OutboundMessageAuditEventRecord): MessageStage { + return event.action === "message.outbound.queued" + ? 0 + : event.action === "message.outbound.platform-started" + ? 1 + : 2; +} + +function compositeMessageRowId(event: OutboundMessageAuditEventRecord): number { + if (event.sequence >= MESSAGE_CURSOR_STAGE_SPAN) { + throw new Error("outbound message decision cursor is outside the supported integer range"); + } + const rowId = messageStage(event) * MESSAGE_CURSOR_STAGE_SPAN + event.sequence; + if (!Number.isSafeInteger(rowId)) { + throw new Error("outbound message decision cursor is outside the supported integer range"); + } + return rowId; +} + +function readTerminalEventsForRun(params: { + runId: string; + contextId?: string; + executionId?: string; + after?: { occurredAt: number; sequence: number }; + limit: number; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): OutboundMessageAuditEventRecord[] { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if (exact && !tableExists(db, "outbound_message_execution_bindings")) { + return []; + } + let query = deliveryAuditDb(db) + .selectFrom("audit_events") + .selectAll() + .where("kind", "=", "message") + .where("direction", "=", "outbound") + .where("action", "=", "message.outbound.finished") + .where("run_id", "=", params.runId) + .where("occurred_at", ">=", (params.now ?? Date.now()) - AUDIT_EVENT_RETENTION_MS); + if (exact) { + query = query.where("event_id", "in", terminalBindingEventIds(db, params.runId, exact)); + } + const after = params.after; + if (after) { + query = query.where((expression) => + expression.or([ + expression("occurred_at", ">", after.occurredAt), + expression.and([ + expression("occurred_at", "=", after.occurredAt), + expression("sequence", ">", after.sequence), + ]), + ]), + ); + } + return executeSqliteQuerySync( + db, + query.orderBy("occurred_at", "asc").orderBy("sequence", "asc").limit(params.limit), + ).rows.map( + // SAFETY: the query fixes the row to the validated outbound terminal-message variant. + (row) => rowToAuditEvent(row) as OutboundMessageAuditEventRecord, + ); + }, params.database) ?? [] + ); +} + +function compareOwnedMessageEvents(left: OwnedMessageEvent, right: OwnedMessageEvent): number { + return left.event.occurredAt === right.event.occurredAt + ? left.rowId - right.rowId + : left.event.occurredAt - right.event.occurredAt; +} + +function streamAfterCursor( + cursor: OutboundMessageAuditEventCursor | undefined, + stage: MessageStage, +): MessageStream["after"] { + if (!cursor) { + return undefined; + } + const cursorStage = Math.floor(cursor.rowId / MESSAGE_CURSOR_STAGE_SPAN); + const cursorSequence = cursor.rowId % MESSAGE_CURSOR_STAGE_SPAN; + return { + occurredAt: cursor.occurredAt, + sequence: + stage < cursorStage ? Number.MAX_SAFE_INTEGER : stage > cursorStage ? 0 : cursorSequence, + }; +} + +function fillMessageStream( + stream: MessageStream, + params: MessageExecutionSelector & { + now: number; + database?: OpenClawStateDatabaseOptions; + }, +): void { + if (stream.buffered.length > 0 || stream.exhausted) { + return; + } + const query = { + runId: params.runId, + ...(params.contextId ? { contextId: params.contextId } : {}), + ...(params.executionId ? { executionId: params.executionId } : {}), + now: params.now, + database: params.database, + after: stream.after, + limit: MESSAGE_STREAM_CHUNK_SIZE, + }; + const events = + stream.stage === 2 + ? readTerminalEventsForRun(query) + : readOutboundMessageProgressForRun({ + ...query, + action: + stream.stage === 0 ? "message.outbound.queued" : "message.outbound.platform-started", + }); + stream.buffered = events.map((event) => ({ event, rowId: compositeMessageRowId(event) })); + stream.exhausted = events.length < MESSAGE_STREAM_CHUNK_SIZE; + const last = events.at(-1); + if (last) { + stream.after = { occurredAt: last.occurredAt, sequence: last.sequence }; + } +} + +function takeNextMessageEvent( + streams: MessageStream[], + params: MessageExecutionSelector & { + now: number; + database?: OpenClawStateDatabaseOptions; + }, +): OwnedMessageEvent | undefined { + for (const stream of streams) { + fillMessageStream(stream, params); + } + let selected: MessageStream | undefined; + for (const stream of streams) { + const candidate = stream.buffered[0]; + const current = selected?.buffered[0]; + if (candidate && (!current || compareOwnedMessageEvents(candidate, current) < 0)) { + selected = stream; + } + } + return selected?.buffered.shift(); +} + +function hasTerminalCursor(params: { + runId: string; + contextId?: string; + executionId?: string; + occurredAt: number; + sequence: number; + database?: OpenClawStateDatabaseOptions; +}): boolean { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if (exact && !tableExists(db, "outbound_message_execution_bindings")) { + return false; + } + let query = deliveryAuditDb(db) + .selectFrom("audit_events") + .select("sequence") + .where("sequence", "=", params.sequence) + .where("run_id", "=", params.runId) + .where("occurred_at", "=", params.occurredAt) + .where("kind", "=", "message") + .where("direction", "=", "outbound") + .where("action", "=", "message.outbound.finished"); + if (exact) { + query = query.where("event_id", "in", terminalBindingEventIds(db, params.runId, exact)); + } + return Boolean(executeSqliteQueryTakeFirstSync(db, query)); + }, params.database) ?? false + ); +} + +/** Count retained owner-native outbound lifecycle records for one run. */ +export function countOutboundMessageAuditEventsForRun(params: { + runId: string; + contextId?: string; + executionId?: string; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): number { + return ( + (withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if (exact && !tableExists(db, "outbound_message_execution_bindings")) { + return 0; + } + let query = deliveryAuditDb(db) + .selectFrom("audit_events") + .select((expression) => expression.fn.countAll().as("count")) + .where("kind", "=", "message") + .where("direction", "=", "outbound") + .where("action", "=", "message.outbound.finished") + .where("run_id", "=", params.runId) + .where("occurred_at", ">=", (params.now ?? Date.now()) - AUDIT_EVENT_RETENTION_MS); + if (exact) { + query = query.where("event_id", "in", terminalBindingEventIds(db, params.runId, exact)); + } + const row = executeSqliteQueryTakeFirstSync(db, query); + return normalizeSqliteNumber(row?.count ?? null) ?? 0; + }, params.database) ?? 0) + countOutboundMessageProgressForRun(params) + ); +} + +/** Page retained owner-native outbound lifecycle records in decision order. */ +export function pageOutboundMessageAuditEventsForRun(params: { + runId: string; + contextId?: string; + executionId?: string; + after?: OutboundMessageAuditEventCursor; + offset?: number; + limit: number; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): { entries: OwnedMessageEvent[]; nextCursor?: OutboundMessageAuditEventCursor } { + if (params.after) { + const stage = Math.floor(params.after.rowId / MESSAGE_CURSOR_STAGE_SPAN); + const sequence = params.after.rowId % MESSAGE_CURSOR_STAGE_SPAN; + const retained = + Number.isSafeInteger(sequence) && sequence >= 1 && stage >= 0 && stage <= 2 + ? stage === 2 + ? hasTerminalCursor({ + runId: params.runId, + ...(params.contextId ? { contextId: params.contextId } : {}), + ...(params.executionId ? { executionId: params.executionId } : {}), + occurredAt: params.after.occurredAt, + sequence, + database: params.database, + }) + : hasOutboundMessageProgressCursor({ + runId: params.runId, + ...(params.contextId ? { contextId: params.contextId } : {}), + ...(params.executionId ? { executionId: params.executionId } : {}), + occurredAt: params.after.occurredAt, + sequence, + action: stage === 0 ? "message.outbound.queued" : "message.outbound.platform-started", + database: params.database, + }) + : false; + if (!retained) { + throw new Error("outbound message decision cursor is no longer retained"); + } + } + const streams: MessageStream[] = ([0, 1, 2] as const).map((stage) => ({ + stage, + after: streamAfterCursor(params.after, stage), + buffered: [], + exhausted: false, + })); + const streamParams = { + runId: params.runId, + ...(params.contextId ? { contextId: params.contextId } : {}), + ...(params.executionId ? { executionId: params.executionId } : {}), + now: params.now ?? Date.now(), + database: params.database, + }; + let remainingOffset = params.offset ?? 0; + while (remainingOffset > 0 && takeNextMessageEvent(streams, streamParams)) { + remainingOffset -= 1; + } + const rows: OwnedMessageEvent[] = []; + while (rows.length <= params.limit) { + const next = takeNextMessageEvent(streams, streamParams); + if (!next) { + break; + } + rows.push(next); + } + const pageRows = rows.slice(0, params.limit); + const last = pageRows.at(-1); + return { + entries: pageRows, + ...(rows.length > params.limit && last + ? { nextCursor: { occurredAt: last.event.occurredAt, rowId: last.rowId } } + : {}), + }; +} diff --git a/src/audit/message-delivery-progress-store.ts b/src/audit/message-delivery-progress-store.ts new file mode 100644 index 0000000000000000000000000000000000000000..97667b5b7af18c95b0ca06716948b1ba3179c851 --- /dev/null +++ b/src/audit/message-delivery-progress-store.ts @@ -0,0 +1,481 @@ +/** Lazy owner-native persistence for nonterminal outbound message progress. */ +import { randomUUID } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { normalizeSqliteNumber } from "../infra/sqlite-number.js"; +import { withExistingOpenClawStateDatabaseReadOnly } from "../state/openclaw-state-db-readonly.js"; +import { ensureColumn, tableExists } from "../state/openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js"; +import type { + OutboundMessageAuditEventRecord, + OutboundMessageProgressInput, +} from "./audit-event-types.js"; +import { + clearAuditIdentityKeyCacheForDatabase, + loadOrCreateAuditIdentityKey, + pseudonymizeAuditIdentity, +} from "./audit-identity.js"; +import { + confirmMessageExecutionBinding, + hasMessageExecutionBindingColumns, + planMessageExecutionBinding, + selectMessageExecutionBinding, + type MessageExecutionBinding, +} from "./message-execution-binding.js"; + +type ProgressTable = OpenClawStateKyselyDatabase["outbound_message_progress"]; +type ProgressDatabase = Pick; +type ProgressRow = Selectable; + +const OUTBOUND_MESSAGE_PROGRESS_RETENTION_MS = 30 * 24 * 60 * 60_000; +const OUTBOUND_MESSAGE_PROGRESS_MAX_ROWS = 200_000; +const OUTBOUND_MESSAGE_PROGRESS_PRUNE_BATCH_ROWS = 1_024; +const AUDIT_HMAC_REF_RE = /^hmac-sha256:v1:[a-f0-9]{32}:[a-f0-9]{64}$/u; +const ensuredDatabases = new WeakSet(); +const progressRowCounts = new WeakMap(); + +function progressSchemaSql(): string { + const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf( + "CREATE TABLE IF NOT EXISTS outbound_message_progress (", + ); + const finalIndex = OPENCLAW_STATE_SCHEMA_SQL.indexOf( + "CREATE INDEX IF NOT EXISTS outbound_message_progress_run_occurred_idx", + start, + ); + const end = finalIndex >= 0 ? OPENCLAW_STATE_SCHEMA_SQL.indexOf(";", finalIndex) : -1; + if (start < 0 || end < 0) { + throw new Error("canonical outbound message progress schema is missing"); + } + return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + 1); +} + +function progressDb(db: DatabaseSync) { + return getNodeSqliteKysely(db); +} + +function ensureProgressSchema(options: OpenClawStateDatabaseOptions): void { + const database = openOpenClawStateDatabase(options); + if (ensuredDatabases.has(database.db)) { + return; + } + runOpenClawStateWriteTransaction( + ({ db }) => { + // sqlite-allow-raw -- feature-local additive schema DDL; progress rows use Kysely. + db.exec(progressSchemaSql()); + ensureColumn(db, "outbound_message_progress", "context_id TEXT"); + ensureColumn(db, "outbound_message_progress", "execution_id TEXT"); + }, + options, + { operationLabel: "audit.outbound-message-progress.schema.ensure" }, + ); + ensuredDatabases.add(database.db); +} + +function projectProgressIdentities(db: DatabaseSync, input: OutboundMessageProgressInput) { + const identity = loadOrCreateAuditIdentityKey(db); + const conversationId = input.conversationId ?? input.targetId; + const ref = ( + kind: Parameters[0]["kind"], + value: string | undefined, + ) => + pseudonymizeAuditIdentity({ + identity, + kind, + channel: input.channel, + ...(kind !== "account" && input.accountId !== undefined + ? { accountId: input.accountId } + : {}), + value, + }); + return { + accountRef: ref("account", input.accountId), + conversationRef: ref("conversation", conversationId), + targetRef: ref("target", input.targetId), + }; +} + +function bindProgressRow( + db: DatabaseSync, + input: OutboundMessageProgressInput, + executionBinding?: MessageExecutionBinding, +): Insertable { + const refs = projectProgressIdentities(db, input); + return { + progress_id: randomUUID(), + source_id: input.sourceId, + source_sequence: input.sourceSequence, + schema_version: 1, + occurred_at: input.occurredAt, + action: input.action, + outcome: input.outcome, + actor_type: input.actorType, + actor_id: input.actorId, + agent_id: input.agentId ?? null, + run_id: input.runId ?? null, + context_id: executionBinding?.contextId ?? null, + execution_id: executionBinding?.executionId ?? null, + channel: input.channel, + conversation_kind: input.conversationKind, + duration_ms: input.durationMs ?? null, + account_ref: refs.accountRef ?? null, + conversation_ref: refs.conversationRef ?? null, + target_ref: refs.targetRef ?? null, + }; +} + +function requiredText(value: unknown, field: string): string { + if (typeof value !== "string" || value.length === 0) { + throw new Error(`corrupt outbound message progress row: invalid ${field}`); + } + return value; +} + +function optionalText(value: unknown, field: string): string | undefined { + if (value === null || value === undefined) { + return undefined; + } + return requiredText(value, field); +} + +function optionalHmacRef(value: unknown, field: string): string | undefined { + const ref = optionalText(value, field); + if (ref !== undefined && !AUDIT_HMAC_REF_RE.test(ref)) { + throw new Error(`corrupt outbound message progress row: invalid ${field}`); + } + return ref; +} + +function rowToProgressEvent(row: ProgressRow): OutboundMessageAuditEventRecord { + const sequence = normalizeSqliteNumber(row.sequence); + const sourceSequence = normalizeSqliteNumber(row.source_sequence); + const occurredAt = normalizeSqliteNumber(row.occurred_at); + const durationMs = normalizeSqliteNumber(row.duration_ms); + if ( + sequence === undefined || + sequence < 1 || + sourceSequence === undefined || + sourceSequence < 1 || + occurredAt === undefined || + occurredAt < 0 || + row.schema_version !== 1 || + (durationMs !== undefined && durationMs < 0) + ) { + throw new Error("corrupt outbound message progress row: invalid numeric field"); + } + if ( + (row.action === "message.outbound.queued" && row.outcome !== "queued") || + (row.action === "message.outbound.platform-started" && row.outcome !== "platform_started") || + (row.action !== "message.outbound.queued" && row.action !== "message.outbound.platform-started") + ) { + throw new Error("corrupt outbound message progress row: invalid lifecycle field"); + } + const actorType = + row.actor_type === "agent" ? "agent" : row.actor_type === "system" ? "system" : undefined; + if (!actorType) { + throw new Error("corrupt outbound message progress row: invalid actor type"); + } + const conversationKind = + row.conversation_kind === "direct" + ? "direct" + : row.conversation_kind === "group" + ? "group" + : row.conversation_kind === "channel" + ? "channel" + : row.conversation_kind === "unknown" + ? "unknown" + : undefined; + if (!conversationKind) { + throw new Error("corrupt outbound message progress row: invalid conversation kind"); + } + const common = { + schemaVersion: 1 as const, + sequence, + eventId: requiredText(row.progress_id, "progressId"), + sourceSequence, + occurredAt, + redaction: "metadata_only" as const, + kind: "message" as const, + actorType, + actorId: requiredText(row.actor_id, "actorId"), + ...(optionalText(row.agent_id, "agentId") !== undefined ? { agentId: row.agent_id! } : {}), + ...(optionalText(row.run_id, "runId") !== undefined ? { runId: row.run_id! } : {}), + direction: "outbound" as const, + channel: requiredText(row.channel, "channel"), + conversationKind, + ...(durationMs !== undefined ? { durationMs } : {}), + resultCount: 0, + ...(optionalHmacRef(row.account_ref, "accountRef") !== undefined + ? { accountRef: row.account_ref! } + : {}), + ...(optionalHmacRef(row.conversation_ref, "conversationRef") !== undefined + ? { conversationRef: row.conversation_ref! } + : {}), + ...(optionalHmacRef(row.target_ref, "targetRef") !== undefined + ? { targetRef: row.target_ref! } + : {}), + } as const; + return row.action === "message.outbound.queued" + ? { ...common, action: row.action, status: "started", outcome: "queued" } + : { ...common, action: row.action, status: "started", outcome: "platform_started" }; +} + +function countProgressRows(db: DatabaseSync): number { + const row = executeSqliteQueryTakeFirstSync( + db, + progressDb(db) + .selectFrom("outbound_message_progress") + .select((expression) => expression.fn.countAll().as("count")), + ); + return normalizeSqliteNumber(row?.count ?? null) ?? 0; +} + +function deleteExpiredProgressRows(db: DatabaseSync, now: number, limit: number) { + const kysely = progressDb(db); + const expiredSequences = kysely + .selectFrom("outbound_message_progress") + .select("sequence") + .where("occurred_at", "<", now - OUTBOUND_MESSAGE_PROGRESS_RETENTION_MS) + .orderBy("occurred_at", "asc") + .orderBy("sequence", "asc") + .limit(limit); + return executeSqliteQuerySync( + db, + kysely.deleteFrom("outbound_message_progress").where("sequence", "in", expiredSequences), + ); +} + +function pruneProgressAfterInsert(db: DatabaseSync, now: number): void { + const kysely = progressDb(db); + const expired = deleteExpiredProgressRows(db, now, OUTBOUND_MESSAGE_PROGRESS_PRUNE_BATCH_ROWS); + const cachedCount = progressRowCounts.get(db); + let rowCount = + cachedCount === undefined + ? countProgressRows(db) + : Math.max(0, cachedCount + 1 - Number(expired.numAffectedRows ?? 0n)); + if (rowCount <= OUTBOUND_MESSAGE_PROGRESS_MAX_ROWS) { + progressRowCounts.set(db, rowCount); + return; + } + const retainedRows = Math.max( + 0, + OUTBOUND_MESSAGE_PROGRESS_MAX_ROWS - OUTBOUND_MESSAGE_PROGRESS_PRUNE_BATCH_ROWS, + ); + const overflow = executeSqliteQueryTakeFirstSync( + db, + kysely + .selectFrom("outbound_message_progress") + .select("sequence") + .orderBy("sequence", "desc") + .offset(retainedRows) + .limit(1), + ); + const cutoff = overflow ? normalizeSqliteNumber(overflow.sequence) : undefined; + if (cutoff !== undefined) { + const pruned = executeSqliteQuerySync( + db, + kysely.deleteFrom("outbound_message_progress").where("sequence", "<=", cutoff), + ); + rowCount = Math.max(0, rowCount - Number(pruned.numAffectedRows ?? 0n)); + } + progressRowCounts.set(db, rowCount); +} + +/** Persist one progress fact idempotently; first use installs only this owner table. */ +export function recordOutboundMessageProgress( + input: OutboundMessageProgressInput, + options: OpenClawStateDatabaseOptions = {}, +): OutboundMessageAuditEventRecord | undefined { + const executionToken = planMessageExecutionBinding(input.executionIdentityToken, input.runId); + ensureProgressSchema(options); + let cacheDatabase: DatabaseSync | undefined; + try { + return runOpenClawStateWriteTransaction(({ db }) => { + cacheDatabase = db; + const executionBinding = confirmMessageExecutionBinding(db, executionToken); + const insert = executeSqliteQuerySync( + db, + progressDb(db) + .insertInto("outbound_message_progress") + .values(bindProgressRow(db, input, executionBinding)) + .onConflict((conflict) => conflict.column("source_id").doNothing()), + ); + if (insert.insertId === undefined) { + return undefined; + } + const sequence = Number(insert.insertId); + if (!Number.isSafeInteger(sequence) || sequence < 1) { + throw new Error("outbound message progress sequence is outside the supported range"); + } + pruneProgressAfterInsert(db, Date.now()); + const row = executeSqliteQueryTakeFirstSync( + db, + progressDb(db) + .selectFrom("outbound_message_progress") + .selectAll() + .where("sequence", "=", sequence), + ); + return row ? rowToProgressEvent(row) : undefined; + }, options); + } catch (error) { + if (cacheDatabase) { + progressRowCounts.delete(cacheDatabase); + clearAuditIdentityKeyCacheForDatabase(cacheDatabase); + } + throw error; + } +} + +export function countOutboundMessageProgressForRun(params: { + runId: string; + contextId?: string; + executionId?: string; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): number { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if ( + !tableExists(db, "outbound_message_progress") || + (exact && !hasMessageExecutionBindingColumns(db, "outbound_message_progress")) + ) { + return 0; + } + let query = progressDb(db) + .selectFrom("outbound_message_progress") + .select((expression) => expression.fn.countAll().as("count")) + .where("run_id", "=", params.runId) + .where( + "occurred_at", + ">=", + (params.now ?? Date.now()) - OUTBOUND_MESSAGE_PROGRESS_RETENTION_MS, + ); + if (exact) { + query = query + .where("context_id", "=", exact.contextId) + .where("execution_id", "=", exact.executionId); + } + const row = executeSqliteQueryTakeFirstSync(db, query); + return normalizeSqliteNumber(row?.count ?? null) ?? 0; + }, params.database) ?? 0 + ); +} + +export function readOutboundMessageProgressForRun(params: { + runId: string; + contextId?: string; + executionId?: string; + action: OutboundMessageProgressInput["action"]; + after?: { occurredAt: number; sequence: number }; + limit: number; + now?: number; + database?: OpenClawStateDatabaseOptions; +}): OutboundMessageAuditEventRecord[] { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if ( + !tableExists(db, "outbound_message_progress") || + (exact && !hasMessageExecutionBindingColumns(db, "outbound_message_progress")) + ) { + return []; + } + let query = progressDb(db) + .selectFrom("outbound_message_progress") + .selectAll() + .where("run_id", "=", params.runId) + .where("action", "=", params.action) + .where( + "occurred_at", + ">=", + (params.now ?? Date.now()) - OUTBOUND_MESSAGE_PROGRESS_RETENTION_MS, + ); + if (exact) { + query = query + .where("context_id", "=", exact.contextId) + .where("execution_id", "=", exact.executionId); + } + const after = params.after; + if (after) { + query = query.where((expression) => + expression.or([ + expression("occurred_at", ">", after.occurredAt), + expression.and([ + expression("occurred_at", "=", after.occurredAt), + expression("sequence", ">", after.sequence), + ]), + ]), + ); + } + return executeSqliteQuerySync( + db, + query.orderBy("occurred_at", "asc").orderBy("sequence", "asc").limit(params.limit), + ).rows.map(rowToProgressEvent); + }, params.database) ?? [] + ); +} + +export function hasOutboundMessageProgressCursor(params: { + runId: string; + contextId?: string; + executionId?: string; + occurredAt: number; + sequence: number; + action: OutboundMessageProgressInput["action"]; + database?: OpenClawStateDatabaseOptions; +}): boolean { + return ( + withExistingOpenClawStateDatabaseReadOnly(({ db }) => { + const exact = selectMessageExecutionBinding(params); + if ( + !tableExists(db, "outbound_message_progress") || + (exact && !hasMessageExecutionBindingColumns(db, "outbound_message_progress")) + ) { + return false; + } + let query = progressDb(db) + .selectFrom("outbound_message_progress") + .select("sequence") + .where("sequence", "=", params.sequence) + .where("run_id", "=", params.runId) + .where("occurred_at", "=", params.occurredAt) + .where("action", "=", params.action); + if (exact) { + query = query + .where("context_id", "=", exact.contextId) + .where("execution_id", "=", exact.executionId); + } + return Boolean(executeSqliteQueryTakeFirstSync(db, query)); + }, params.database) ?? false + ); +} + +/** Prune existing progress without creating its lazy table. */ +export function pruneExpiredOutboundMessageProgress( + params: { now?: number; database?: OpenClawStateDatabaseOptions } = {}, +): number { + const database = openOpenClawStateDatabase(params.database); + if (!tableExists(database.db, "outbound_message_progress")) { + return 0; + } + return runOpenClawStateWriteTransaction(({ db }) => { + const deleted = deleteExpiredProgressRows( + db, + params.now ?? Date.now(), + OUTBOUND_MESSAGE_PROGRESS_PRUNE_BATCH_ROWS, + ); + progressRowCounts.delete(db); + return Number(deleted.numAffectedRows ?? 0n); + }, params.database); +} diff --git a/src/audit/message-delivery-receipts.ts b/src/audit/message-delivery-receipts.ts new file mode 100644 index 0000000000000000000000000000000000000000..e3ee7be0b096287eca8d9f1506f167b3fdaa5693 --- /dev/null +++ b/src/audit/message-delivery-receipts.ts @@ -0,0 +1,175 @@ +/** Owner-native outbound message lifecycle projection for run inspection. */ +import type { + DecisionReceiptV1, + ExecutionIdentityContextV1, +} from "../../packages/gateway-protocol/src/index.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import type { OutboundMessageAuditEventRecord } from "./audit-event-types.js"; +import { + countOutboundMessageAuditEventsForRun, + pageOutboundMessageAuditEventsForRun, + type OutboundMessageAuditEventCursor, +} from "./message-delivery-audit-store.js"; + +type MessageDeliveryReadOptions = OpenClawStateDatabaseOptions & { now?: number }; + +function messageOutcome( + event: OutboundMessageAuditEventRecord, +): Pick { + switch (event.outcome) { + case "queued": + return { + decision: { outcome: "allowed", reasonCode: "message_queued" }, + remediation: [ + { + code: "inspect_delivery_progress", + text: "Inspect this run again to observe the platform and terminal delivery outcome.", + }, + ], + }; + case "platform_started": + return { + decision: { outcome: "allowed", reasonCode: "message_platform_started" }, + remediation: [ + { + code: "inspect_delivery_result", + text: "Inspect this run again for a delivered, failed, or unknown terminal outcome.", + }, + ], + }; + case "sent": + return { + decision: { outcome: "allowed", reasonCode: "message_delivered" }, + remediation: [], + }; + case "suppressed": + return { + decision: { + outcome: "not-applicable", + reasonCode: `message_suppressed_${event.reasonCode}`, + }, + remediation: [ + { + code: "revise_suppressed_message", + text: "Revise the outbound content or remove the suppressing hook before retrying.", + }, + ], + }; + case "failed": + return { + decision: { + outcome: "unknown", + reasonCode: `message_delivery_failed_${event.failureStage}`, + }, + remediation: [ + { + code: "inspect_delivery_failure", + text: "Inspect channel configuration and delivery logs before retrying the message.", + }, + ], + }; + case "unknown": + return { + decision: { + outcome: "unknown", + reasonCode: `message_delivery_unknown_${event.failureStage}`, + }, + remediation: [ + { + code: "reconcile_delivery_outcome", + text: "Reconcile the platform delivery outcome before retrying to avoid a duplicate message.", + }, + ], + }; + } + throw new Error("unsupported outbound message outcome"); +} + +function projectMessageDeliveryReceipt( + event: OutboundMessageAuditEventRecord, + context: ExecutionIdentityContextV1, +): DecisionReceiptV1 { + const resourceRef = `channel:${event.channel}`; + const outcome = messageOutcome(event); + return { + schemaVersion: 1, + receiptId: `message:${event.eventId}`, + contextId: context.contextId, + executionId: context.executionId, + runId: context.runId, + actionId: event.eventId, + occurredAt: event.occurredAt, + action: { + family: "message", + operation: "send", + ...(resourceRef.length <= 256 ? { resourceRef } : {}), + ...(event.targetRef ? { targetRef: event.targetRef } : {}), + summary: `Outbound message lifecycle: ${event.outcome.replaceAll("_", "-")}.`, + }, + decision: outcome.decision, + enforcement: { + coverageState: "attribution-only", + evaluatorRef: "outbound-delivery", + policyRefs: [], + grantRefs: [], + contextFieldsUsed: ["contextId", "executionId", "runId"], + }, + source: { + owner: + event.action === "message.outbound.finished" ? "audit_events" : "outbound_message_progress", + recordRef: event.eventId, + decisionBoundary: event.action, + }, + // Exact linkage remains diagnostic provenance. Delivery never becomes an + // authorization claim merely because it belongs to this execution. + missingEvidence: [], + remediation: outcome.remediation, + }; +} + +export function summarizeMessageDeliveryReceiptsForRun(params: { + context: ExecutionIdentityContextV1; + options: MessageDeliveryReadOptions; +}): { count: number; coverageState?: "attribution-only"; missingEvidence: string[] } { + const count = countOutboundMessageAuditEventsForRun({ + runId: params.context.runId, + contextId: params.context.contextId, + executionId: params.context.executionId, + now: params.options.now, + database: params.options, + }); + return { + count, + ...(count > 0 ? { coverageState: "attribution-only" as const } : {}), + missingEvidence: [], + }; +} + +export function pageMessageDeliveryReceiptsForRun(params: { + context: ExecutionIdentityContextV1; + after?: OutboundMessageAuditEventCursor; + offset?: number; + limit: number; + options: MessageDeliveryReadOptions; +}): { + entries: Array<{ receipt: DecisionReceiptV1; selectorId: string }>; + nextCursor?: OutboundMessageAuditEventCursor; +} { + const page = pageOutboundMessageAuditEventsForRun({ + runId: params.context.runId, + contextId: params.context.contextId, + executionId: params.context.executionId, + after: params.after, + offset: params.offset, + limit: params.limit, + now: params.options.now, + database: params.options, + }); + return { + entries: page.entries.map(({ event, rowId }) => ({ + receipt: projectMessageDeliveryReceipt(event, params.context), + selectorId: `message-decision:${rowId}`, + })), + ...(page.nextCursor ? { nextCursor: page.nextCursor } : {}), + }; +} diff --git a/src/audit/message-execution-binding.ts b/src/audit/message-execution-binding.ts new file mode 100644 index 0000000000000000000000000000000000000000..ed7a19ac85ded38c9273d1c1f7ba60b5a022a173 --- /dev/null +++ b/src/audit/message-execution-binding.ts @@ -0,0 +1,156 @@ +/** Exact execution binding for owner-native outbound message lifecycle facts. */ +import type { DatabaseSync } from "node:sqlite"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "../infra/kysely-sync.js"; +import { tableExists, tableHasColumn } from "../state/openclaw-state-db-schema-helpers.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, + type OpenClawStateDatabaseOptions, +} from "../state/openclaw-state-db.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.js"; +import { + parseExecutionIdentityAdmissionToken, + type ExecutionIdentityAdmissionToken, +} from "./execution-identity-admission.js"; + +type ExecutionBindingDatabase = Pick< + OpenClawStateKyselyDatabase, + "execution_identity_contexts" | "outbound_message_execution_bindings" +>; +const ensuredTerminalBindingDatabases = new WeakSet(); + +export type MessageExecutionBinding = Readonly<{ + contextId: string; + executionId: string; +}>; + +export function selectMessageExecutionBinding(params: { + contextId?: string; + executionId?: string; +}): MessageExecutionBinding | undefined { + if (params.contextId === undefined && params.executionId === undefined) { + return undefined; + } + if (!params.contextId || !params.executionId) { + throw new Error("outbound message decision query requires the exact context and execution"); + } + return { contextId: params.contextId, executionId: params.executionId }; +} + +export function hasMessageExecutionBindingColumns( + db: DatabaseSync, + tableName: "outbound_message_progress", +): boolean { + return ( + tableHasColumn(db, tableName, "context_id") && tableHasColumn(db, tableName, "execution_id") + ); +} + +function terminalBindingSchemaSql(): string { + const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf( + "CREATE TABLE IF NOT EXISTS outbound_message_execution_bindings (", + ); + const indexStart = OPENCLAW_STATE_SCHEMA_SQL.indexOf( + "CREATE INDEX IF NOT EXISTS outbound_message_execution_bindings_execution_event_idx", + start, + ); + const end = indexStart >= 0 ? OPENCLAW_STATE_SCHEMA_SQL.indexOf(";", indexStart) : -1; + if (start < 0 || end < 0) { + throw new Error("canonical outbound message execution binding schema is missing"); + } + return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + 1); +} + +/** Install the terminal binding companion only when an exact producer first uses it. */ +export function ensureTerminalMessageExecutionBindingSchema( + options: OpenClawStateDatabaseOptions, +): void { + const database = openOpenClawStateDatabase(options); + if (ensuredTerminalBindingDatabases.has(database.db)) { + return; + } + runOpenClawStateWriteTransaction( + ({ db }) => { + // sqlite-allow-raw -- feature-local additive schema DDL; bindings use Kysely. + db.exec(terminalBindingSchemaSql()); + }, + { ...options, database }, + { operationLabel: "audit.outbound-message.execution-binding.schema.ensure" }, + ); + ensuredTerminalBindingDatabases.add(database.db); +} + +/** Validate queue-loaded token bytes before entering a synchronous write transaction. */ +export function planMessageExecutionBinding( + token: ExecutionIdentityAdmissionToken | undefined, + runId: string | undefined, +): ExecutionIdentityAdmissionToken | undefined { + if (!token) { + return undefined; + } + const planned = parseExecutionIdentityAdmissionToken(token); + if (!runId || planned.runId !== runId) { + throw new Error("outbound message execution binding disagrees with the admitted run"); + } + return planned; +} + +/** Confirm the exact retained admission row; run correlation alone never binds a receipt. */ +export function confirmMessageExecutionBinding( + db: DatabaseSync, + token: ExecutionIdentityAdmissionToken | undefined, +): MessageExecutionBinding | undefined { + if (!token || !tableExists(db, "execution_identity_contexts")) { + return undefined; + } + const row = executeSqliteQueryTakeFirstSync( + db, + getNodeSqliteKysely(db) + .selectFrom("execution_identity_contexts") + .select("context_id") + .where("context_id", "=", token.contextId) + .where("execution_id", "=", token.executionId) + .where("run_id", "=", token.runId) + .where("created_at", "=", token.createdAt), + ); + return row ? { contextId: token.contextId, executionId: token.executionId } : undefined; +} + +function recordTerminalMessageExecutionBinding( + db: DatabaseSync, + params: MessageExecutionBinding & { eventId: string; runId: string }, +): void { + executeSqliteQuerySync( + db, + getNodeSqliteKysely(db) + .insertInto("outbound_message_execution_bindings") + .values({ + event_id: params.eventId, + context_id: params.contextId, + execution_id: params.executionId, + run_id: params.runId, + }), + ); +} + +export function recordConfirmedTerminalMessageExecutionBinding( + db: DatabaseSync, + params: { eventId: string | undefined; token?: ExecutionIdentityAdmissionToken }, +): void { + if (!params.eventId) { + return; + } + const binding = confirmMessageExecutionBinding(db, params.token); + if (binding && params.token) { + recordTerminalMessageExecutionBinding(db, { + eventId: params.eventId, + runId: params.token.runId, + ...binding, + }); + } +} diff --git a/src/canvas/config.ts b/src/canvas/config.ts new file mode 100644 index 0000000000000000000000000000000000000000..9b7380df74f86c876cdecd9bad13b082b8463e5c --- /dev/null +++ b/src/canvas/config.ts @@ -0,0 +1,18 @@ +/** Core Canvas host enablement from the shipped Canvas plugin configuration surface. */ +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isTruthyEnvValue } from "../infra/env.js"; + +/** Returns whether core-owned widget hosting and tools should be active. */ +export function isCoreCanvasHostEnabled( + config?: OpenClawConfig, + env: NodeJS.ProcessEnv = process.env, +): boolean { + // Canvas owned these shipped operator switches before hosting moved into core. + // Core keeps reading them so existing disablement still covers the whole Canvas family. + if (isTruthyEnvValue(env.OPENCLAW_SKIP_CANVAS_HOST)) { + return false; + } + const host = config?.plugins?.entries?.canvas?.config?.host; + return !isRecord(host) || host.enabled !== false; +} diff --git a/src/canvas/constants.test.ts b/src/canvas/constants.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c7ca7e9dabc1b0cc71bdcb1a2444502035642fc6 --- /dev/null +++ b/src/canvas/constants.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { + isCanvasDocumentHttpPath, + resolveCanvasNodeCapability, + withCoreCanvasNodeCapability, +} from "./constants.js"; + +describe("core canvas contracts", () => { + it("keeps document auth scoped to the stable Canvas path", () => { + expect(isCanvasDocumentHttpPath("/__openclaw__/canvas/documents/cv_1/index.html")).toBe(true); + expect(isCanvasDocumentHttpPath("/__openclaw__/canvas/index.html")).toBe(false); + expect( + resolveCanvasNodeCapability([ + "/encoded-path", + "/__openclaw__/canvas/documents/cv_1/index.html", + ]), + ).toEqual({ surface: "canvas", scopeKey: "canvas:canvas" }); + }); + + it("advertises one core-owned canvas surface with the historical scope key", () => { + expect( + withCoreCanvasNodeCapability([ + { surface: "canvas", scopeKey: "canvas:canvas" }, + { surface: "files", scopeKey: "files:files" }, + ]), + ).toEqual([ + { surface: "canvas", scopeKey: "canvas:canvas" }, + { surface: "files", scopeKey: "files:files" }, + ]); + }); +}); diff --git a/src/canvas/constants.ts b/src/canvas/constants.ts new file mode 100644 index 0000000000000000000000000000000000000000..f2358e38d015945ef3d700aedf7ca2ebb10f0f3b --- /dev/null +++ b/src/canvas/constants.ts @@ -0,0 +1,34 @@ +import type { PluginNodeCapabilitySurface } from "../gateway/plugin-node-capability.js"; + +/** Stable Gateway path used by chat clients for hosted Canvas content. */ +const CANVAS_HOST_PATH = "/__openclaw__/canvas"; +export const CANVAS_DOCUMENTS_PATH = `${CANVAS_HOST_PATH}/documents`; + +/** Keep the historical Canvas plugin scope so existing capability URLs remain valid. */ +const CANVAS_NODE_CAPABILITY: PluginNodeCapabilitySurface = { + surface: "canvas", + scopeKey: "canvas:canvas", +}; + +/** Returns true only for the core-owned Canvas document subtree. */ +export function isCanvasDocumentHttpPath(pathname: string): boolean { + return pathname.startsWith(`${CANVAS_DOCUMENTS_PATH}/`); +} + +/** Resolves auth for any canonicalized candidate targeting core Canvas documents. */ +export function resolveCanvasNodeCapability( + pathCandidates: readonly string[], +): PluginNodeCapabilitySurface | undefined { + return pathCandidates.some(isCanvasDocumentHttpPath) ? CANVAS_NODE_CAPABILITY : undefined; +} + +/** Adds the core Canvas surface while removing stale plugin-owned duplicates. */ +export function withCoreCanvasNodeCapability( + surfaces: readonly PluginNodeCapabilitySurface[], + enabled = true, +): PluginNodeCapabilitySurface[] { + const withoutStaleCanvas = surfaces.filter( + (entry) => entry.surface.trim() !== CANVAS_NODE_CAPABILITY.surface, + ); + return enabled ? [CANVAS_NODE_CAPABILITY, ...withoutStaleCanvas] : withoutStaleCanvas; +} diff --git a/src/canvas/documents.test.ts b/src/canvas/documents.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..56de9185bfd0c0e947d50aba0950da67f5576f3d --- /dev/null +++ b/src/canvas/documents.test.ts @@ -0,0 +1,270 @@ +// Core Canvas document storage and URL contract coverage. +import { link, mkdir, readFile, rename, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { + createCanvasDocument, + readCanvasDocumentHtmlSource, + resolveCanvasDocumentsDir, + resolveCanvasHttpPathToLocalPath, +} from "./documents.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +function resolveCanvasDocumentDir(stateDir: string, documentId: string): string { + return path.join(resolveCanvasDocumentsDir(stateDir), documentId); +} + +describe("canvas documents", () => { + it.skipIf(process.platform === "win32").each(["document", "manifest.json", "index.html"])( + "rejects a symlinked %s when reading widget HTML", + async (target) => { + const stateDir = tempDirs.make("openclaw-canvas-links-"); + const outsideDir = tempDirs.make("openclaw-canvas-outside-"); + const document = await createCanvasDocument( + { kind: "html_bundle", entrypoint: { type: "html", value: "

outside

" } }, + { stateDir }, + ); + const documentDir = resolveCanvasDocumentDir(stateDir, document.id); + const original = target === "document" ? documentDir : path.join(documentDir, target); + const outside = path.join(outsideDir, target); + await rename(original, outside); + await symlink(outside, original, target === "document" ? "dir" : "file"); + + await expect(readCanvasDocumentHtmlSource(document.id, { stateDir })).rejects.toMatchObject({ + code: target === "document" ? "outside-workspace" : "symlink", + }); + }, + ); + + it.each(["manifest.json", "index.html"])( + "rejects a hardlinked %s when reading widget HTML", + async (target) => { + const stateDir = tempDirs.make("openclaw-canvas-hardlinks-"); + const document = await createCanvasDocument( + { kind: "html_bundle", entrypoint: { type: "html", value: "

aliased

" } }, + { stateDir }, + ); + await link( + path.join(resolveCanvasDocumentDir(stateDir, document.id), target), + path.join(stateDir, `alias-${target}`), + ); + await expect(readCanvasDocumentHtmlSource(document.id, { stateDir })).rejects.toMatchObject({ + code: "hardlink", + }); + }, + ); + + it("bounds HTML reads by bytes while independently allowing the manifest", async () => { + const stateDir = tempDirs.make("openclaw-canvas-bounded-"); + const document = await createCanvasDocument( + { kind: "html_bundle", entrypoint: { type: "html", value: "éééé" } }, + { stateDir }, + ); + await expect( + readCanvasDocumentHtmlSource(document.id, { stateDir, maxBytes: 7 }), + ).rejects.toMatchObject({ code: "too-large" }); + await expect( + readCanvasDocumentHtmlSource(document.id, { stateDir, maxBytes: 8 }), + ).resolves.toEqual({ html: "éééé" }); + }); + + it("rejects oversized manifests before parsing them", async () => { + const stateDir = tempDirs.make("openclaw-canvas-manifest-"); + const document = await createCanvasDocument( + { kind: "html_bundle", entrypoint: { type: "html", value: "

small

" } }, + { stateDir }, + ); + const manifestPath = path.join( + resolveCanvasDocumentDir(stateDir, document.id), + "manifest.json", + ); + await writeFile( + manifestPath, + JSON.stringify({ ...document, title: "x".repeat(2 * 1024 * 1024) }), + ); + await expect(readCanvasDocumentHtmlSource(document.id, { stateDir })).rejects.toMatchObject({ + code: "too-large", + }); + }); + + it("builds entry urls for materialized path documents under managed storage", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const workspaceDir = tempDirs.make("openclaw-canvas-documents-workspace-"); + await mkdir(path.join(workspaceDir, "player"), { recursive: true }); + await writeFile(path.join(workspaceDir, "player/index.html"), "
ok
", "utf8"); + + const document = await createCanvasDocument( + { + kind: "html_bundle", + entrypoint: { type: "path", value: "player/index.html" }, + }, + { stateDir, workspaceDir }, + ); + + expect(document.entryUrl).toContain("/__openclaw__/canvas/documents/"); + expect(document.localEntrypoint).toBe("index.html"); + expect(resolveCanvasDocumentDir(stateDir, document.id)).toContain(stateDir); + }); + + it("materializes inline html bundles as index documents", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const document = await createCanvasDocument( + { + kind: "html_bundle", + title: "Preview", + entrypoint: { + type: "html", + value: + "
Front
", + }, + }, + { stateDir }, + ); + + const indexHtml = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, document.id), "index.html"), + "utf8", + ); + expect(indexHtml).toContain("
Front
"); + expect(indexHtml).toContain(""); + expect(document.title).toBe("Preview"); + expect(document.entryUrl).toBe(`/__openclaw__/canvas/documents/${document.id}/index.html`); + await expect(readCanvasDocumentHtmlSource(document.id, { stateDir })).resolves.toEqual({ + html: indexHtml, + }); + }); + + it("reports the document sandbox policy alongside board source bytes", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const document = await createCanvasDocument( + { + kind: "html_bundle", + entrypoint: { type: "html", value: "" }, + cspSandbox: "scripts", + }, + { stateDir }, + ); + + await expect(readCanvasDocumentHtmlSource(document.id, { stateDir })).resolves.toEqual({ + html: "", + cspSandbox: "scripts", + }); + }); + + it("reuses a supplied stable id by replacing the prior materialized view", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const first = await createCanvasDocument( + { + id: "status-card", + kind: "html_bundle", + entrypoint: { type: "html", value: "
first
" }, + }, + { stateDir }, + ); + const second = await createCanvasDocument( + { + id: "status-card", + kind: "html_bundle", + entrypoint: { type: "html", value: "
second
" }, + }, + { stateDir }, + ); + + expect(first.id).toBe("status-card"); + expect(second.id).toBe("status-card"); + const indexHtml = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, second.id), "index.html"), + "utf8", + ); + expect(indexHtml).toContain("second"); + expect(indexHtml).not.toContain("first"); + }); + + it("copies declared assets into managed storage", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const workspaceDir = tempDirs.make("openclaw-canvas-documents-workspace-"); + await mkdir(path.join(workspaceDir, "collection.media"), { recursive: true }); + await writeFile(path.join(workspaceDir, "collection.media/audio.mp3"), "audio", "utf8"); + + const document = await createCanvasDocument( + { + kind: "html_bundle", + entrypoint: { type: "html", value: "" }, + assets: [ + { + logicalPath: "collection.media/audio.mp3", + sourcePath: "collection.media/audio.mp3", + contentType: "audio/mpeg", + }, + ], + }, + { stateDir, workspaceDir }, + ); + + expect(document.assets).toEqual([ + { logicalPath: "collection.media/audio.mp3", contentType: "audio/mpeg" }, + ]); + await expect( + readFile( + path.join(resolveCanvasDocumentDir(stateDir, document.id), "collection.media/audio.mp3"), + "utf8", + ), + ).resolves.toBe("audio"); + }); + + it("wraps local and remote PDF documents in index viewer pages", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + const workspaceDir = tempDirs.make("openclaw-canvas-documents-workspace-"); + await writeFile(path.join(workspaceDir, "demo.pdf"), "%PDF-1.4", "utf8"); + const localDocument = await createCanvasDocument( + { kind: "document", entrypoint: { type: "path", value: "demo.pdf" } }, + { stateDir, workspaceDir }, + ); + const remoteDocument = await createCanvasDocument( + { + kind: "document", + entrypoint: { type: "url", value: "https://example.com/demo.pdf" }, + }, + { stateDir }, + ); + + const localHtml = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, localDocument.id), "index.html"), + "utf8", + ); + const remoteHtml = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, remoteDocument.id), "index.html"), + "utf8", + ); + expect(localHtml).toContain('data="demo.pdf"'); + expect(remoteHtml).toContain('data="https://example.com/demo.pdf"'); + }); + + it("rejects traversal and malformed encoded hosted paths", async () => { + const stateDir = tempDirs.make("openclaw-canvas-documents-"); + expect( + resolveCanvasHttpPathToLocalPath( + "/__openclaw__/canvas/documents/../collection.media/index.html", + { stateDir }, + ), + ).toBeNull(); + + const documentDir = resolveCanvasDocumentDir(stateDir, "cv_malformed"); + await mkdir(documentDir, { recursive: true }); + await writeFile(path.join(documentDir, "%E0%A4%A.html"), "literal-percent-name", "utf8"); + expect( + resolveCanvasHttpPathToLocalPath( + "/__openclaw__/canvas/documents/cv_malformed/%E0%A4%A.html", + { stateDir }, + ), + ).toBeNull(); + expect( + resolveCanvasHttpPathToLocalPath( + "/__openclaw__/canvas/documents/cv_malformed/%25E0%25A4%25A.html", + { stateDir }, + ), + ).toBe(path.join(documentDir, "%E0%A4%A.html")); + }); +}); diff --git a/src/canvas/documents.ts b/src/canvas/documents.ts new file mode 100644 index 0000000000000000000000000000000000000000..9cf3a613df0dbcb8304589c875540756bd26212a --- /dev/null +++ b/src/canvas/documents.ts @@ -0,0 +1,379 @@ +/** Core Canvas document materialization and hosted-path resolution. */ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { sanitizeUntrustedFileName } from "../infra/fs-safe-advanced.js"; +import { root as fsRoot } from "../infra/fs-safe.js"; +import { escapeHtml } from "../shared/html-escape.js"; +import { resolveUserPath } from "../utils.js"; +import { CANVAS_DOCUMENTS_PATH } from "./constants.js"; + +const CANVAS_DOCUMENT_READ_MAX_BYTES = 2 * 1024 * 1024; + +type CanvasDocumentKind = "html_bundle" | "url_embed" | "document" | "image" | "video_asset"; + +type CanvasDocumentAsset = { + logicalPath: string; + sourcePath: string; + contentType?: string; +}; + +type CanvasDocumentEntrypoint = + | { type: "html"; value: string } + | { type: "path"; value: string } + | { type: "url"; value: string }; + +type CanvasDocumentCreateInput = { + id?: string; + kind: CanvasDocumentKind; + title?: string; + preferredHeight?: number; + entrypoint?: CanvasDocumentEntrypoint; + assets?: CanvasDocumentAsset[]; + surface?: "assistant_message" | "tool_card" | "sidebar"; + retentionScope?: string; + /** Serve with a CSP sandbox header so direct opens get an opaque origin. */ + cspSandbox?: "scripts"; +}; + +export type CanvasDocumentManifest = { + id: string; + kind: CanvasDocumentKind; + title?: string; + preferredHeight?: number; + createdAt: string; + entryUrl: string; + localEntrypoint?: string; + externalUrl?: string; + surface?: "assistant_message" | "tool_card" | "sidebar"; + retentionScope?: string; + cspSandbox?: "scripts"; + assets: Array<{ + logicalPath: string; + contentType?: string; + }>; +}; + +function isPdfPathLike(value: string): boolean { + return /\.pdf(?:[?#].*)?$/i.test(value.trim()); +} + +function buildPdfWrapper(url: string): string { + const escaped = escapeHtml(url); + return `

Unable to render PDF preview. Open PDF.

`; +} + +function hasControlCharacter(value: string): boolean { + for (const char of value) { + const code = char.charCodeAt(0); + if (code < 0x20 || code === 0x7f) { + return true; + } + } + return false; +} + +function normalizeLogicalPath(value: string): string { + const normalized = value.replaceAll("\\", "/").replace(/^\/+/, ""); + const parts = normalized.split("/").filter(Boolean); + if ( + parts.length === 0 || + parts.some( + (part) => part === "." || part === ".." || part.includes(":") || hasControlCharacter(part), + ) + ) { + throw new Error("canvas document logicalPath invalid"); + } + return parts.join("/"); +} + +function canvasDocumentId(): string { + return `cv_${randomUUID().replaceAll("-", "")}`; +} + +function normalizeCanvasDocumentId(value: string): string { + const normalized = value.trim(); + if ( + !normalized || + normalized === "." || + normalized === ".." || + !/^[A-Za-z0-9._-]+$/.test(normalized) + ) { + throw new Error("canvas document id invalid"); + } + return normalized; +} + +/** Stable root for existing and newly created Canvas documents. */ +export function resolveCanvasDocumentsDir(stateDir = resolveStateDir()): string { + return path.resolve(stateDir, "canvas", "documents"); +} + +/** Reads the managed HTML entrypoint for a core Canvas document. */ +export async function readCanvasDocumentHtmlSource( + documentId: string, + options?: { stateDir?: string; maxBytes?: number }, +): Promise<{ html: string; cspSandbox?: "scripts" }> { + const id = normalizeCanvasDocumentId(documentId); + // Keep the document directory inside the guarded root so aliases cannot select another document. + const root = await fsRoot(resolveCanvasDocumentsDir(options?.stateDir), { + maxBytes: CANVAS_DOCUMENT_READ_MAX_BYTES, + }); + const manifest = await root.readJson>(`${id}/manifest.json`); + if (manifest.id !== id || typeof manifest.localEntrypoint !== "string") { + throw new Error(`canvas document has no local entrypoint: ${id}`); + } + const entrypoint = normalizeLogicalPath(manifest.localEntrypoint); + if (!entrypoint.toLowerCase().endsWith(".html")) { + throw new Error(`canvas document entrypoint is not HTML: ${id}`); + } + return { + html: await root.readText(`${id}/${entrypoint}`, { + maxBytes: options?.maxBytes ?? CANVAS_DOCUMENT_READ_MAX_BYTES, + }), + ...(manifest.cspSandbox === "scripts" ? { cspSandbox: "scripts" as const } : {}), + }; +} + +async function pruneCanvasDocumentsForScope(params: { + documentsDir: string; + retentionScope: string; + maxDocuments: number; +}): Promise { + const entries = await fs.readdir(params.documentsDir, { withFileTypes: true }); + const scopedDocuments = ( + await Promise.all( + entries + .filter((entry) => entry.isDirectory()) + .map(async (entry) => { + try { + const manifest = JSON.parse( + await fs.readFile( + path.join(params.documentsDir, entry.name, "manifest.json"), + "utf8", + ), + ) as { createdAt?: unknown; retentionScope?: unknown }; + if ( + manifest.retentionScope !== params.retentionScope || + typeof manifest.createdAt !== "string" + ) { + return null; + } + return { id: entry.name, createdAt: manifest.createdAt }; + } catch { + return null; + } + }), + ) + ).filter((entry): entry is { id: string; createdAt: string } => entry !== null); + const deleteCount = Math.max(0, scopedDocuments.length - params.maxDocuments); + const oldest = scopedDocuments + .toSorted( + (left, right) => + left.createdAt.localeCompare(right.createdAt) || left.id.localeCompare(right.id), + ) + .slice(0, deleteCount); + await Promise.all( + oldest.map((entry) => + fs.rm(path.join(params.documentsDir, entry.id), { recursive: true, force: true }), + ), + ); +} + +/** Resolves the on-disk directory for one Canvas document id. */ +function resolveCanvasDocumentDir(documentId: string, options?: { stateDir?: string }): string { + return path.join(resolveCanvasDocumentsDir(options?.stateDir), documentId); +} + +/** Builds the hosted URL path for a Canvas document entrypoint. */ +function buildCanvasDocumentEntryUrl(documentId: string, entrypoint: string): string { + const normalizedEntrypoint = normalizeLogicalPath(entrypoint); + const encodedEntrypoint = normalizedEntrypoint + .split("/") + .map((segment) => encodeURIComponent(segment)) + .join("/"); + return `${CANVAS_DOCUMENTS_PATH}/${encodeURIComponent(documentId)}/${encodedEntrypoint}`; +} + +/** Maps a Canvas hosted document URL path back to its managed local file. */ +export function resolveCanvasHttpPathToLocalPath( + requestPath: string, + options?: { stateDir?: string }, +): string | null { + const trimmed = requestPath.trim(); + const prefix = `${CANVAS_DOCUMENTS_PATH}/`; + if (!trimmed.startsWith(prefix)) { + return null; + } + const pathWithoutQuery = trimmed.replace(/[?#].*$/, ""); + const relative = pathWithoutQuery.slice(prefix.length); + const segments: string[] = []; + for (const segment of relative.split("/")) { + if (!segment) { + continue; + } + try { + segments.push(decodeURIComponent(segment)); + } catch { + return null; + } + } + if (segments.length < 2) { + return null; + } + const [rawDocumentId, ...entrySegments] = segments; + if (!rawDocumentId) { + return null; + } + try { + const documentId = normalizeCanvasDocumentId(rawDocumentId); + const normalizedEntrypoint = normalizeLogicalPath(entrySegments.join("/")); + const documentsDir = resolveCanvasDocumentsDir(options?.stateDir); + const candidatePath = path.resolve( + resolveCanvasDocumentDir(documentId, options), + normalizedEntrypoint, + ); + if (!candidatePath.startsWith(`${documentsDir}${path.sep}`)) { + return null; + } + return candidatePath; + } catch { + return null; + } +} + +type CanvasDocumentRoot = Awaited>; + +async function copyAssets( + root: CanvasDocumentRoot, + assets: CanvasDocumentAsset[] | undefined, + workspaceDir: string, +): Promise { + const copied: CanvasDocumentManifest["assets"] = []; + for (const asset of assets ?? []) { + const logicalPath = normalizeLogicalPath(asset.logicalPath); + const sourcePath = asset.sourcePath.startsWith("~") + ? resolveUserPath(asset.sourcePath) + : path.isAbsolute(asset.sourcePath) + ? path.resolve(asset.sourcePath) + : path.resolve(workspaceDir, asset.sourcePath); + await root.copyIn(logicalPath, sourcePath); + copied.push({ + logicalPath, + ...(asset.contentType ? { contentType: asset.contentType } : {}), + }); + } + return copied; +} + +async function materializeEntrypoint( + rootDir: string, + root: CanvasDocumentRoot, + input: CanvasDocumentCreateInput, + workspaceDir: string, +): Promise> { + const entrypoint = input.entrypoint; + if (!entrypoint) { + throw new Error("canvas document entrypoint required"); + } + if (entrypoint.type === "html") { + const fileName = "index.html"; + await root.write(fileName, entrypoint.value); + return { + localEntrypoint: fileName, + entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), fileName), + }; + } + if (entrypoint.type === "url") { + if (input.kind === "document" && isPdfPathLike(entrypoint.value)) { + const fileName = "index.html"; + await root.write(fileName, buildPdfWrapper(entrypoint.value)); + return { + localEntrypoint: fileName, + externalUrl: entrypoint.value, + entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), fileName), + }; + } + return { externalUrl: entrypoint.value, entryUrl: entrypoint.value }; + } + + const resolvedPath = entrypoint.value.startsWith("~") + ? resolveUserPath(entrypoint.value) + : path.isAbsolute(entrypoint.value) + ? path.resolve(entrypoint.value) + : path.resolve(workspaceDir, entrypoint.value); + + if (input.kind === "image" || input.kind === "video_asset") { + const copiedName = sanitizeUntrustedFileName(path.basename(resolvedPath), "asset"); + await root.copyIn(copiedName, resolvedPath); + const wrapper = + input.kind === "image" + ? `` + : ``; + await root.write("index.html", wrapper); + return { + localEntrypoint: "index.html", + entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), "index.html"), + }; + } + + const fileName = sanitizeUntrustedFileName(path.basename(resolvedPath), "document"); + await root.copyIn(fileName, resolvedPath); + if (input.kind === "document" && isPdfPathLike(fileName)) { + await root.write("index.html", buildPdfWrapper(fileName)); + return { + localEntrypoint: "index.html", + entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), "index.html"), + }; + } + return { + localEntrypoint: fileName, + entryUrl: buildCanvasDocumentEntryUrl(path.basename(rootDir), fileName), + }; +} + +/** Creates a Canvas document directory, copies assets, and writes its manifest. */ +export async function createCanvasDocument( + input: CanvasDocumentCreateInput, + options?: { + stateDir?: string; + workspaceDir?: string; + maxDocumentsPerScope?: number; + }, +): Promise { + const workspaceDir = options?.workspaceDir ?? process.cwd(); + const id = input.id?.trim() ? normalizeCanvasDocumentId(input.id) : canvasDocumentId(); + const rootDir = resolveCanvasDocumentDir(id, { stateDir: options?.stateDir }); + await fs.rm(rootDir, { recursive: true, force: true }).catch(() => undefined); + await fs.mkdir(rootDir, { recursive: true }); + const root = await fsRoot(rootDir); + const assets = await copyAssets(root, input.assets, workspaceDir); + const entry = await materializeEntrypoint(rootDir, root, input, workspaceDir); + const manifest: CanvasDocumentManifest = { + id, + kind: input.kind, + ...(input.title?.trim() ? { title: input.title.trim() } : {}), + ...(typeof input.preferredHeight === "number" + ? { preferredHeight: input.preferredHeight } + : {}), + ...(input.surface ? { surface: input.surface } : {}), + ...(input.retentionScope ? { retentionScope: input.retentionScope } : {}), + ...(input.cspSandbox ? { cspSandbox: input.cspSandbox } : {}), + createdAt: new Date().toISOString(), + entryUrl: entry.entryUrl, + ...(entry.localEntrypoint ? { localEntrypoint: entry.localEntrypoint } : {}), + ...(entry.externalUrl ? { externalUrl: entry.externalUrl } : {}), + assets, + }; + await root.writeJson("manifest.json", manifest, { space: 2 }); + if (input.retentionScope && options?.maxDocumentsPerScope) { + // Bounded transcript widgets cannot grow managed Canvas storage without limit. + await pruneCanvasDocumentsForScope({ + documentsDir: resolveCanvasDocumentsDir(options.stateDir), + retentionScope: input.retentionScope, + maxDocuments: options.maxDocumentsPerScope, + }); + } + return manifest; +} diff --git a/src/canvas/serve.runtime.test.ts b/src/canvas/serve.runtime.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..594490154e3fcc4b5b789df4e2e66af9ddd588c4 --- /dev/null +++ b/src/canvas/serve.runtime.test.ts @@ -0,0 +1,194 @@ +// Core Canvas document HTTP response coverage. +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import type { IncomingMessage, ServerResponse } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createCanvasDocument, resolveCanvasDocumentsDir } from "./documents.js"; +import { handleCanvasDocumentHttpRequest } from "./serve.runtime.js"; +import { buildWidgetDocument } from "./wrap.js"; + +const tempDirs: string[] = []; + +afterEach(async () => { + vi.unstubAllEnvs(); + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function createStateDir(): Promise { + const stateDir = await mkdtemp(path.join(tmpdir(), "openclaw-canvas-serve-")); + tempDirs.push(stateDir); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + return stateDir; +} + +async function capture(url: string, method = "GET") { + const response = { + statusCode: 200, + headers: {} as Record, + body: Buffer.alloc(0) as Buffer, + setHeader(name: string, value: number | string | readonly string[]) { + this.headers[name.toLowerCase()] = typeof value === "object" ? [...value] : value; + return this; + }, + end(chunk?: string | Buffer) { + this.body = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk ?? ""); + return this; + }, + }; + const handled = await handleCanvasDocumentHttpRequest( + { method, url } as IncomingMessage, + response as unknown as ServerResponse, + ); + return { handled, ...response, text: response.body.toString("utf8") }; +} + +describe("core canvas document host", () => { + it.each([ + [ + "stored", + ``, + ], + [ + "new", + buildWidgetDocument( + "Registered renderer", + '', + { scriptOrigins: ["'self'"] }, + ), + ], + ])("preserves document-approved renderer sources for %s documents", async (_, html) => { + const stateDir = await createStateDir(); + const document = await createCanvasDocument( + { kind: "html_bundle", entrypoint: { type: "html", value: html }, cspSandbox: "scripts" }, + { stateDir }, + ); + const response = await capture(document.entryUrl); + const policy = String(response.headers["content-security-policy"]); + const scripts = policy + .split(";") + .find((directive) => directive.trim().startsWith("script-src ")); + expect(scripts?.split(/\s+/)).toContain("'self'"); + expect(policy).toContain("connect-src 'none'"); + expect(policy).toContain("sandbox allow-scripts"); + expect(response.text).toBe(html); + }); + + it("serves sandbox-marked HTML with the stable CSP header and no mutation", async () => { + const stateDir = await createStateDir(); + const html = "widget"; + const document = await createCanvasDocument( + { + id: "widget-1", + kind: "html_bundle", + entrypoint: { type: "html", value: html }, + cspSandbox: "scripts", + }, + { stateDir }, + ); + + const response = await capture(document.entryUrl); + expect(response.handled).toBe(true); + expect(response.statusCode).toBe(200); + expect(response.headers["content-type"]).toBe("text/html; charset=utf-8"); + expect(response.headers["cache-control"]).toBe("no-store"); + const csp = String(response.headers["content-security-policy"]); + expect(csp).toContain("sandbox allow-scripts"); + expect(csp).toContain("default-src 'none'"); + expect(csp).toContain("https://cdn.jsdelivr.net"); + expect(csp).toContain("font-src data:"); + expect(csp).toContain("connect-src 'none'"); + expect(response.headers["referrer-policy"]).toBe("no-referrer"); + expect(response.text).toBe(html); + }); + + it("omits the sandbox response header for unmarked documents", async () => { + const stateDir = await createStateDir(); + const document = await createCanvasDocument( + { + id: "plain-1", + kind: "html_bundle", + entrypoint: { type: "html", value: "plain" }, + }, + { stateDir }, + ); + + const response = await capture(document.entryUrl); + expect(response.statusCode).toBe(200); + expect(response.headers["content-security-policy"]).toBeUndefined(); + }); + + it("serves Content-Length on HEAD responses", async () => { + const stateDir = await createStateDir(); + const html = "widget"; + const document = await createCanvasDocument( + { + id: "widget-1", + kind: "html_bundle", + entrypoint: { type: "html", value: html }, + cspSandbox: "scripts", + }, + { stateDir }, + ); + const css = "body { color: red; }"; + await writeFile( + path.join(resolveCanvasDocumentsDir(stateDir), "widget-1", "style.css"), + css, + "utf8", + ); + const cssUrl = document.entryUrl.replace(/index\.html$/, "style.css"); + + const getHtml = await capture(document.entryUrl); + const headHtml = await capture(document.entryUrl, "HEAD"); + expect(headHtml.statusCode).toBe(200); + expect(headHtml.headers["content-length"]).toBe(String(getHtml.body.byteLength)); + expect(headHtml.body.byteLength).toBe(0); + expect(headHtml.headers["content-type"]).toBe("text/html; charset=utf-8"); + expect(headHtml.headers["content-security-policy"]).toBe( + getHtml.headers["content-security-policy"], + ); + + const getCss = await capture(cssUrl); + const headCss = await capture(cssUrl, "HEAD"); + expect(getCss.statusCode).toBe(200); + expect(headCss.statusCode).toBe(200); + expect(headCss.headers["content-length"]).toBe(String(getCss.body.byteLength)); + expect(headCss.body.byteLength).toBe(0); + + // Invalid UTF-8 in a copied HTML file expands to U+FFFD when served, so the + // header must be measured from the decoded representation, not raw bytes. + const brokenBytes = Buffer.from([ + 0x3c, 0x68, 0x31, 0x3e, 0xff, 0xfe, 0x3c, 0x2f, 0x68, 0x31, 0x3e, + ]); + await writeFile( + path.join(resolveCanvasDocumentsDir(stateDir), "widget-1", "broken.html"), + brokenBytes, + ); + const brokenUrl = document.entryUrl.replace(/index\.html$/, "broken.html"); + const getBroken = await capture(brokenUrl); + const headBroken = await capture(brokenUrl, "HEAD"); + expect(getBroken.statusCode).toBe(200); + expect(getBroken.body.byteLength).toBeGreaterThan(brokenBytes.byteLength); + expect(headBroken.headers["content-length"]).toBe(String(getBroken.body.byteLength)); + expect(headBroken.body.byteLength).toBe(0); + }); + + it("rejects unsupported methods and traversal paths", async () => { + await createStateDir(); + const methodResponse = await capture( + "/__openclaw__/canvas/documents/widget-1/index.html", + "POST", + ); + expect(methodResponse.statusCode).toBe(405); + expect(methodResponse.text).toBe("Method Not Allowed"); + + const traversalResponse = await capture( + "/__openclaw__/canvas/documents/../widget-1/index.html", + ); + expect(traversalResponse.handled).toBe(false); + + const missingResponse = await capture("/__openclaw__/canvas/documents/widget-1/index.html"); + expect(missingResponse.statusCode).toBe(404); + expect(missingResponse.text).toBe("not found"); + }); +}); diff --git a/src/canvas/serve.runtime.ts b/src/canvas/serve.runtime.ts new file mode 100644 index 0000000000000000000000000000000000000000..147762118e239ed067837705accb0bd08d52cb53 --- /dev/null +++ b/src/canvas/serve.runtime.ts @@ -0,0 +1,122 @@ +/** Gateway runtime for core-owned Canvas document HTTP responses. */ +import type { IncomingMessage, ServerResponse } from "node:http"; +import path from "node:path"; +import { detectMime } from "@openclaw/media-core/mime"; +import { buildBoardWidgetContentSecurityPolicy } from "../gateway/board-sandbox.js"; +import { FsSafeError, root as fsRoot } from "../infra/fs-safe.js"; +import { + resolveCanvasDocumentsDir, + resolveCanvasHttpPathToLocalPath, + type CanvasDocumentManifest, +} from "./documents.js"; + +async function readRootFile(root: Awaited>, relativePath: string) { + try { + const opened = await root.open(relativePath); + try { + return { data: await opened.handle.readFile(), realPath: opened.realPath }; + } finally { + await opened.handle.close().catch(() => {}); + } + } catch (error) { + if (error instanceof FsSafeError) { + return null; + } + throw error; + } +} + +async function resolveDocumentSandbox( + root: Awaited>, + relativePath: string, +): Promise<"scripts" | undefined> { + const documentId = relativePath.split(path.sep)[0]; + if (!documentId) { + return undefined; + } + const opened = await readRootFile(root, path.join(documentId, "manifest.json")); + if (!opened) { + return undefined; + } + try { + const manifest = JSON.parse(opened.data.toString("utf8")) as CanvasDocumentManifest; + return manifest.cspSandbox === "scripts" ? "scripts" : undefined; + } catch { + return undefined; + } +} + +/** Serves one managed Canvas document request. */ +export async function handleCanvasDocumentHttpRequest( + req: IncomingMessage, + res: ServerResponse, +): Promise { + const localPath = resolveCanvasHttpPathToLocalPath(req.url ?? ""); + if (!localPath) { + return false; + } + if (req.method !== "GET" && req.method !== "HEAD") { + res.statusCode = 405; + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end("Method Not Allowed"); + return true; + } + + try { + const documentsDir = resolveCanvasDocumentsDir(); + const relativePath = path.relative(documentsDir, localPath); + const root = await fsRoot(documentsDir); + const opened = await readRootFile(root, relativePath); + if (!opened) { + res.statusCode = 404; + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end("not found"); + return true; + } + + const lowerPath = opened.realPath.toLowerCase(); + const mime = + lowerPath.endsWith(".html") || lowerPath.endsWith(".htm") + ? "text/html" + : ((await detectMime({ filePath: opened.realPath })) ?? "application/octet-stream"); + res.setHeader("Cache-Control", "no-store"); + if (mime === "text/html") { + // Measure the decoded representation: toString("utf8") expands invalid + // bytes to U+FFFD, so the raw file length can differ from the body sent. + const body = opened.data.toString("utf8"); + res.setHeader("Content-Type", "text/html; charset=utf-8"); + res.setHeader("Content-Length", String(Buffer.byteLength(body))); + if ((await resolveDocumentSandbox(root, relativePath)) === "scripts") { + // Registered documents allow local renderer scripts in their own CSP; + // the response policy must preserve that permission because policies intersect. + res.setHeader( + "Content-Security-Policy", + buildBoardWidgetContentSecurityPolicy({ + grantState: "none", + resourceOrigins: ["'self'"], + }), + ); + res.setHeader("Referrer-Policy", "no-referrer"); + } + if (req.method === "HEAD") { + res.end(); + return true; + } + res.end(body); + return true; + } + res.setHeader("Content-Type", mime); + res.setHeader("Content-Length", String(opened.data.byteLength)); + if (req.method === "HEAD") { + res.end(); + return true; + } + res.end(opened.data); + return true; + } catch (error) { + res.statusCode = error instanceof FsSafeError ? 404 : 500; + res.setHeader("Content-Type", "text/plain; charset=utf-8"); + res.end(error instanceof FsSafeError ? "not found" : "error"); + return true; + } +} diff --git a/src/canvas/widget-script-syntax.test.ts b/src/canvas/widget-script-syntax.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..d30854527eb29f1e4b4c08848a2e3a9b2e48f967 --- /dev/null +++ b/src/canvas/widget-script-syntax.test.ts @@ -0,0 +1,243 @@ +import { describe, expect, it } from "vitest"; +import { findWidgetScriptSyntaxError } from "./widget-script-syntax.js"; + +describe("widget script syntax", () => { + it("maps a raw newline in a single-quoted string to the original widget line", () => { + const widgetCode = "\n
\n \n
"; + expect(findWidgetScriptSyntaxError(widgetCode)).toEqual({ + message: "Unterminated string constant", + line: 3, + column: 18, + snippet: "', + '', + '', + '', + '', + '', + "", + "', + "const =", + "1 < 2 ", + "", + "", + '', + '', + '', + '', + ])("accepts valid scripts or markup: %s", (widgetCode) => { + expect(findWidgetScriptSyntaxError(widgetCode)).toBeUndefined(); + }); + + it.each([ + "", + "type", + 'type=""', + "type='text/javascript'", + "type=application/javascript", + 'TYPE=" TEXT/ECMASCRIPT "', + "type=application/ecmascript", + "type=application/x-javascript", + "type=text/jscript", + "type=text/javascript1.5", + "type=text/livescript", + ])("rejects top-level await in classic scripts: %s", (attributes) => { + expect( + findWidgetScriptSyntaxError(``), + ).toMatchObject({ scriptIndex: 1 }); + }); + + it("does not allow wrapped-function grammar", () => { + expect(findWidgetScriptSyntaxError("")).toBeDefined(); + }); + + it.each([ + 'type="application/json"', + "type='application/ld+json'", + "type=importmap", + "type=text/template", + "type=text/x-handlebars-template", + 'src="script.js"', + "src='script.js'", + "SRC=script.js", + "src", + ])("skips non-JavaScript and external scripts: %s", (attributes) => { + expect(findWidgetScriptSyntaxError(``)).toBeUndefined(); + }); + + it.each(["", ''])( + "maps the second script after %s and stops at a case-insensitive raw-text close", + (first) => { + const widgetCode = `${first}\r\n

Text

\r\n";`; + expect(findWidgetScriptSyntaxError(widgetCode)).toEqual({ + message: "Unterminated string constant", + line: 3, + column: 18, + snippet: '";', + scriptIndex: 2, + }); + }, + ); + + it("reports only the first error across multiple scripts", () => { + expect( + findWidgetScriptSyntaxError(""), + ).toMatchObject({ scriptIndex: 1, line: 1, column: 14 }); + }); + + it.each([ + "", + "", + "", + '
', + " ", + ].join("\n"); + expect(findWidgetScriptSyntaxError(widgetCode)).toEqual({ + message: "Unexpected token", + scriptIndex: 1, + line: 4, + column: 16, + snippet: "", + }); + }); + + it.each(["", '', "", ""])( + "parses the script body through its terminator or EOF: %s", + (endTag) => { + expect(findWidgetScriptSyntaxError(``, + ), + ).toMatchObject({ scriptIndex: 2, line: 2, column: 14 }); + } + }, + ); + + it.each([ + '";', + '";', + ])("ends script data unless the exact script token double-escapes it: %s", (widgetCode) => { + expect(findWidgetScriptSyntaxError(widgetCode)?.message).toBe("Unterminated string constant"); + }); + + it.each([ + "", + "", + "", + '', + "", + "
Example: > <script>const =</script>
", + '
', + "", + '', + ])("parses CDATA-wrapped scripts in SVG: %s", (widgetCode) => { + expect(findWidgetScriptSyntaxError(widgetCode)).toBeUndefined(); + }); + + it.each([ + { + widgetCode: "", + line: 1, + column: 28, + snippet: "", + }, + { + widgetCode: "\n \n", + line: 2, + column: 27, + snippet: "", + }, + { + widgetCode: "\r\n", + line: 4, + column: 6, + snippet: "const =", + }, + ])( + "maps CDATA errors to the original widget: $widgetCode", + ({ widgetCode, line, column, snippet }) => { + expect(findWidgetScriptSyntaxError(widgetCode)).toEqual({ + message: "Unexpected token", + scriptIndex: 1, + line, + column, + snippet, + }); + }, + ); + + it.each([ + "", + "", + '', + "text", + "", + "", + ])("still reaches scripts after self-closing or CDATA foreign content: %s", (widgetCode) => { + expect(findWidgetScriptSyntaxError(widgetCode)).toMatchObject({ line: 1 }); + }); + + it.each(["", "", "", ""])( + "leaves CDATA markers outside SVG untouched after %s", + (prefix) => { + expect( + findWidgetScriptSyntaxError(`${prefix}`), + ).toMatchObject({ message: "Unexpected token", scriptIndex: 1 }); + }, + ); + + it.each(["text/javascript", "text/javascript", "module"])( + "validates scripts whose type contains character references: %s", + (type) => { + expect(findWidgetScriptSyntaxError(``)).toMatchObject({ + message: "Unexpected token", + scriptIndex: 1, + }); + }, + ); + + it("bounds the offending line snippet", () => { + const widgetCode = ``; + expect(findWidgetScriptSyntaxError(widgetCode)?.snippet).toBe(widgetCode.slice(0, 160)); + }); +}); diff --git a/src/canvas/widget-script-syntax.ts b/src/canvas/widget-script-syntax.ts new file mode 100644 index 0000000000000000000000000000000000000000..9d429bbb7cf8f35ccaf894ebb4d9a2d6f3aacd00 --- /dev/null +++ b/src/canvas/widget-script-syntax.ts @@ -0,0 +1,220 @@ +import { getLineInfo, parse } from "acorn"; +import { decodeHTMLAttribute } from "entities"; + +type WidgetScriptSyntaxError = { + message: string; + /** Location in the supplied widget_code: 1-based line, 0-based UTF-16 column. */ + line: number; + column: number; + snippet: string; + /** 1-based index among all scanned script elements, including skipped scripts. */ + scriptIndex: number; +}; + +/** JavaScript MIME type essences per https://mimesniff.spec.whatwg.org/#javascript-mime-type. */ +const JAVASCRIPT_MIME_ESSENCES = new Set([ + "application/ecmascript", + "application/javascript", + "application/x-ecmascript", + "application/x-javascript", + "text/ecmascript", + "text/javascript", + "text/javascript1.0", + "text/javascript1.1", + "text/javascript1.2", + "text/javascript1.3", + "text/javascript1.4", + "text/javascript1.5", + "text/jscript", + "text/livescript", + "text/x-ecmascript", + "text/x-javascript", +]); + +function scriptSourceType(attributes: string): "script" | "module" | undefined { + let type: string | undefined; + const attributePattern = /([^\s=/>]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+)))?/g; + for (const attribute of attributes.matchAll(attributePattern)) { + const name = (attribute[1] ?? "").toLowerCase(); + if (name === "src") { + return undefined; + } + if (name === "type") { + type ??= decodeHTMLAttribute(attribute[2] ?? attribute[3] ?? attribute[4] ?? "") + .trim() + .toLowerCase(); + } + } + if (type === "module") { + return "module"; + } + return !type || JAVASCRIPT_MIME_ESSENCES.has(type) ? "script" : undefined; +} + +/** Finds a tag's closing bracket without interpreting quoted attribute values as markup. */ +function findTagEnd(html: string, offset: number): number { + let quote = ""; + for (let index = offset; index < html.length; index++) { + const char = html[index]; + if (quote) { + if (char === quote) { + quote = ""; + } + } else if (char === '"' || char === "'") { + quote = char; + } else if (char === ">") { + return index; + } + } + return html.length; +} + +/** + * Script-looking text in double-escaped data cannot close the script element, and inside + * foreign (SVG) content a CDATA section keeps everything up to `]]>` as script text. + */ +function findRawTextEnd(html: string, start: number, name: string, foreign: boolean): number { + const tokens = + name === "script" + ? foreign + ? /||<\/?script(?=[\t\n\f\r />])/gi + : /|<\/?script(?=[\t\n\f\r />])/gi + : new RegExp(`])`, "gi"); + tokens.lastIndex = start; + let state: "data" | "escaped" | "double-escaped" | "cdata" = "data"; + for (const match of html.matchAll(tokens)) { + const token = match[0].toLowerCase(); + if (state === "cdata") { + if (token === "]]>") { + state = "data"; + } + } else if (token === "") { + state = "data"; + } else if (token.startsWith("]*)/iy; + let position = 0; + let scriptIndex = 0; + let svgDepth = 0; + let foreignObjectDepth = 0; + while (position < widgetCode.length) { + const start = widgetCode.indexOf("<", position); + if (start < 0) { + break; + } + position = start + 1; + if (widgetCode.startsWith("", start + 4); + position = end < 0 ? widgetCode.length : end + 3; + continue; + } + // foreignObject is an HTML integration point, so its children follow HTML rules again. + const foreign = svgDepth > 0 && foreignObjectDepth === 0; + if (foreign && widgetCode.startsWith("", start + 9); + position = end < 0 ? widgetCode.length : end + 3; + continue; + } + tagPattern.lastIndex = start; + const tag = tagPattern.exec(widgetCode); + if (!tag) { + if (widgetCode[position] === "!" || widgetCode[position] === "/") { + position = findTagEnd(widgetCode, position + 1) + 1; + } + continue; + } + const attributesStart = tagPattern.lastIndex; + const tagEnd = findTagEnd(widgetCode, attributesStart); + position = tagEnd + 1; + if (tagEnd === widgetCode.length) { + continue; + } + const name = (tag[1] ?? "").toLowerCase(); + const closing = widgetCode[start + 1] === "/"; + const selfClosing = widgetCode[tagEnd - 1] === "/"; + if (name === "svg") { + svgDepth = closing ? Math.max(0, svgDepth - 1) : svgDepth + (selfClosing ? 0 : 1); + } else if (name === "foreignobject" && svgDepth > 0) { + foreignObjectDepth = closing + ? Math.max(0, foreignObjectDepth - 1) + : foreignObjectDepth + (selfClosing ? 0 : 1); + } + if (closing) { + continue; + } + if (name === "plaintext") { + break; + } + if (!/^(?:script|style|textarea|title|xmp|iframe|noembed|noframes|noscript)$/.test(name)) { + continue; + } + // Foreign content honors self-closing start tags; HTML content ignores the slash. + if (foreign && selfClosing) { + continue; + } + let bodyStart = position; + const bodyEnd = findRawTextEnd(widgetCode, bodyStart, name, foreign); + const closingBracket = bodyEnd < widgetCode.length ? widgetCode.indexOf(">", bodyEnd) : -1; + position = closingBracket < 0 ? widgetCode.length : closingBracket + 1; + if (name !== "script") { + continue; + } + scriptIndex++; + const sourceType = scriptSourceType(widgetCode.slice(attributesStart, tagEnd)); + if (!sourceType) { + continue; + } + let body = widgetCode.slice(bodyStart, bodyEnd); + const trimmed = body.trim(); + if (foreign && trimmed.startsWith("")) { + bodyStart += body.length - body.trimStart().length + "".length); + } + try { + parse(body, { ecmaVersion: "latest", sourceType }); + } catch (error) { + if (!(error instanceof SyntaxError) || !("pos" in error) || typeof error.pos !== "number") { + throw error; + } + const offset = bodyStart + error.pos; + const { line, column } = getLineInfo(widgetCode, offset); + const snippet = (widgetCode.slice(offset - column).split(/[\r\n\u2028\u2029]/u, 1)[0] ?? "") + .trim() + .slice(0, 160); + return { + message: error.message.replace(/ \(\d+:\d+\)$/u, "").slice(0, 200), + line, + column, + snippet, + scriptIndex, + }; + } + } + return undefined; +} diff --git a/src/canvas/widget-tool.content-kinds.test-support.ts b/src/canvas/widget-tool.content-kinds.test-support.ts new file mode 100644 index 0000000000000000000000000000000000000000..ee6d4501419986396ae008004ff28dab18b5ef4f --- /dev/null +++ b/src/canvas/widget-tool.content-kinds.test-support.ts @@ -0,0 +1,30 @@ +import { createPluginBoardWidgetContentKindRegistrar } from "../plugins/board-widget-content-kinds.js"; +import { createPluginRecord } from "../plugins/loader-records.js"; +import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; +import { setActivePluginRegistry } from "../plugins/runtime.js"; + +export function registerTestWidgetContentKind( + kind = "diagram", + composeDocument = ({ source }: { source: string }) => `
${source}
`, +): void { + const registry = createEmptyPluginRegistry(); + const record = createPluginRecord({ + id: "diagram", + source: "diagram-fixture", + origin: "bundled", + enabled: true, + configSchema: false, + }); + createPluginBoardWidgetContentKindRegistrar(registry)(record, { + kind, + label: "Diagram", + resources: { surface: "diagram", paths: ["/__openclaw__/diagram/app.js"] }, + validateSource(source) { + if (!source.startsWith("diagram:")) { + throw new Error("diagram prefix required"); + } + }, + composeDocument, + }); + setActivePluginRegistry(registry); +} diff --git a/src/canvas/widget-tool.grants.test.ts b/src/canvas/widget-tool.grants.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..6a1ceba28848d8fd93453494dc235f385a240ebf --- /dev/null +++ b/src/canvas/widget-tool.grants.test.ts @@ -0,0 +1,43 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { resetPluginRuntimeStateForTest } from "../plugins/runtime.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { createBoardPutCaller } from "./widget-tool.test-support.js"; + +beforeEach(() => resetPluginRuntimeStateForTest()); +afterEach(() => resetPluginRuntimeStateForTest()); + +describe("show_widget capability outcomes", () => { + it.each(["pending", "rejected", "granted"] as const)( + "reports the owner's %s capability outcome", + async (grantState) => { + const { mock, callGateway } = createBoardPutCaller(); + const original = mock.getMockImplementation()!; + mock.mockImplementation(async (...args) => { + const snapshot = await original(...args); + snapshot.widgets[0]!.grantState = grantState; + return snapshot; + }); + const tool = createShowWidgetTool({ + agentSessionKey: "agent:main:approval", + inlineHostEnabled: false, + callGateway, + }); + const result = await tool.execute("pin", { + title: "Runs", + widget_code: "

runs

", + pin: true, + capabilities: { tools: ["github.actions.runs:owner/repo"] }, + }); + const text = result.content.find((item) => item.type === "text")?.text; + if (!text) { + throw new Error("expected widget tool text result"); + } + const payload = JSON.parse(text); + expect(payload).toMatchObject({ boardWidgetName: "runs", capabilityState: grantState }); + expect(payload.text).toContain(grantState); + if (grantState !== "granted") { + expect(payload.text).toMatch(/approve|review/i); + } + }, + ); +}); diff --git a/src/canvas/widget-tool.presenter.test.ts b/src/canvas/widget-tool.presenter.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..fd856ea7e43ff1c3401f295d98c7c3de2a66c063 --- /dev/null +++ b/src/canvas/widget-tool.presenter.test.ts @@ -0,0 +1,78 @@ +import { access } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import type { WidgetPresenter } from "../plugins/plugin-registration.types.js"; +import { resolveCanvasDocumentsDir } from "./documents.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { buildWidgetDocument } from "./wrap.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("show_widget current-channel presentation", () => { + it.each([false, true])("prefers the channel presenter (inline=%s)", async (inlineClient) => { + const stateDir = tempDirs.make("openclaw-widget-presenter-"); + const present = vi.fn(async () => ({ + ok: true as const, + value: { + kind: "message" as const, + receipt: { + primaryPlatformMessageId: "message-1", + platformMessageIds: ["message-1"], + parts: [], + sentAt: 1, + }, + }, + })); + const context = { + messageChannel: "discord", + accountId: "work", + nativeChannelId: "channel-1", + currentChannelId: "channel:channel-1", + currentMessagingTarget: "discord:channel:channel-1", + sessionKey: "agent:main:discord", + }; + const presenter: WidgetPresenter = { + target: "current_channel", + description: "Post in the current channel", + capabilities: { sourceKinds: ["html"], maxSourceBytes: 48 * 1024 }, + match: (candidate) => candidate.messageChannel === "discord", + availability: async () => ({ ok: true, value: { available: true } }), + present, + }; + const tool = createShowWidgetTool({ + stateDir, + sessionId: "current-channel", + inlineClientAvailable: inlineClient, + presenters: [presenter], + presenterContext: context, + }); + + expect(tool.requiredClientCaps).toBeUndefined(); + expect(tool.description).toContain("current channel presenter"); + expect(tool.description).not.toContain("Keep one-off visualizations inline"); + expect( + (tool.parameters as { properties?: { kind?: { enum?: string[] } } }).properties?.kind?.enum, + ).toEqual(["html"]); + const result = await tool.execute("current-channel", { + title: "Status", + widget_code: "

ready

", + }); + const parsed = JSON.parse(result.content[0]?.type === "text" ? result.content[0].text : "null"); + + expect(parsed).toMatchObject({ + kind: "widget", + presentation: { + target: "current_channel", + title: "Status", + receipt: { primaryPlatformMessageId: "message-1" }, + }, + text: "Widget presented in the current channel as message message-1", + }); + expect(present).toHaveBeenCalledExactlyOnceWith({ + document: { kind: "html", html: buildWidgetDocument("Status", "

ready

") }, + title: "Status", + context, + }); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); +}); diff --git a/src/canvas/widget-tool.prompt.test.ts b/src/canvas/widget-tool.prompt.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..649aef7fdcf347544870ba423f409cb2b1d0926c --- /dev/null +++ b/src/canvas/widget-tool.prompt.test.ts @@ -0,0 +1,132 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createPluginGatewayMethodDescriptor } from "../gateway/methods/descriptor.js"; +import { registerPluginDashboardCapabilities } from "../plugins/dashboard-capabilities.js"; +import { createPluginRecord } from "../plugins/loader-records.js"; +import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; +import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js"; +import * as processExec from "../process/exec.js"; +import { createShowWidgetTool } from "./widget-tool.js"; + +describe("show_widget prompt", () => { + afterEach(() => { + resetPluginRuntimeStateForTest(); + vi.restoreAllMocks(); + }); + + it("discovers active host capabilities with usable contracts", () => { + const registry = createEmptyPluginRegistry(); + const handler = () => {}; + registry.gatewayHandlers["fixture.list"] = handler; + registry.gatewayHandlers["fixture.dispatch"] = handler; + registry.gatewayMethodDescriptors.push( + createPluginGatewayMethodDescriptor({ + pluginId: "fixture", + name: "fixture.list", + scope: "operator.read", + handler, + }), + createPluginGatewayMethodDescriptor({ + pluginId: "fixture", + name: "fixture.dispatch", + scope: "operator.write", + handler, + }), + ); + registerPluginDashboardCapabilities({ + registry, + record: createPluginRecord({ + id: "fixture", + source: "fixture", + origin: "bundled", + enabled: true, + configSchema: false, + dashboard: { + dataBindings: [ + { id: "large", method: "fixture.list", description: "Too large ".repeat(500) }, + { id: "list", method: "fixture.list", description: "List fixture items" }, + ], + actionVerbs: [ + { + id: "dispatch", + method: "fixture.dispatch", + description: "Dispatch items", + paramShape: { type: "object", properties: { force: { type: "boolean" } } }, + }, + ], + }, + }), + }); + setActivePluginRegistry(registry); + const tool = createShowWidgetTool(); + const instructions = JSON.stringify(tool.parameters) + tool.description; + expect(instructions).toContain("With a usable connected agent GitHub identity"); + expect(instructions).toContain("Identity is checked before save"); + expect(instructions).toContain("github.actions.runs"); + expect(instructions).toContain("github.actions.runs:owner/repo"); + expect(instructions).toContain("fixture.list"); + expect(instructions).toContain("List fixture items"); + expect(instructions).toContain("fixture.dispatch"); + expect(instructions).toContain("force"); + expect(instructions).toContain("1 plugin capabilities omitted"); + const guidance = ( + tool.parameters as { + properties: { capabilities: { properties: { tools: { description: string } } } }; + } + ).properties.capabilities.properties.tools.description; + expect(guidance.length).toBeLessThanOrEqual(1200); + expect(guidance.indexOf("fixture.dispatch")).toBeLessThan(guidance.indexOf("fixture.list")); + expect(guidance).not.toContain("Too large"); + setActivePluginRegistry(createEmptyPluginRegistry()); + expect(JSON.stringify(createShowWidgetTool())).not.toContain("fixture.list"); + expect(JSON.stringify(createShowWidgetTool())).not.toContain("fixture.dispatch"); + }); + it("constructs synchronously without probing GitHub identity", () => { + const http = vi + .spyOn(globalThis, "fetch") + .mockRejectedValue(new Error("Unexpected HTTP probe")); + const native = vi + .spyOn(processExec, "runCommandBuffered") + .mockRejectedValue(new Error("Unexpected native probe")); + expect(createShowWidgetTool().name).toBe("show_widget"); + expect(http).not.toHaveBeenCalled(); + expect(native).not.toHaveBeenCalled(); + }); + it("keeps proactive single visualizations inline unless dashboard use meets its threshold", () => { + const tool = createShowWidgetTool(); + const directoryDescription = tool.description.slice(0, 184); + const properties = ( + tool.parameters as { + properties?: { + pin?: { description?: string }; + name?: { description?: string }; + widget_code?: { description?: string }; + }; + } + ).properties; + const pinDescription = properties?.pin?.description; + + expect(directoryDescription).toMatch(/^Visual helps\? Make widget\. Do not wait for ask\./); + expect(directoryDescription).toMatch( + /(?:single|one[- ]off|ad hoc).{0,40}visualizations?.{0,40}inline/i, + ); + expect(directoryDescription).toContain("explicit dashboard request"); + expect(directoryDescription).toContain("multiple non-code visualizations"); + expect(directoryDescription).toContain("Update pinned HTML by name"); + expect(pinDescription).toContain("explicit dashboard request"); + expect(pinDescription).toContain("multiple non-code visualizations"); + expect(properties?.name?.description).toMatch(/same name.*pin=true.*widget_code/i); + expect(properties?.widget_code?.description).toContain("fluid widths"); + expect(properties?.widget_code?.description).toMatch(/wrap or stack.*narrow/i); + }); + + it("offers native reports for a session dashboard with a bounded data contract", () => { + const tool = createShowWidgetTool({ agentSessionKey: "agent:main:report" }); + const description = tool.description; + expect(description).toContain("Prefer the report argument with pin=true"); + expect(description).toContain("dashboard-only"); + const schema = JSON.stringify(tool.parameters); + expect(schema).toContain("Maximum 8KB JSON"); + expect(schema).toContain("Metric values and table cells are strings"); + expect(createShowWidgetTool().description).not.toContain("Prefer the report argument"); + }); +}); diff --git a/src/canvas/widget-tool.report.test.ts b/src/canvas/widget-tool.report.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..1e3232b2cece5dd95478cb6860486a46b4f48b48 --- /dev/null +++ b/src/canvas/widget-tool.report.test.ts @@ -0,0 +1,256 @@ +import { access } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { createDashboardTool } from "../agents/tools/dashboard-tool.js"; +import type { InProcessGatewayCaller } from "../agents/tools/in-process-gateway.js"; +import type { BoardReport } from "../boards/board-report.js"; +import { readBoardHtml, createTestBoardStore } from "../boards/board-store.test-support.js"; +import { createBoardHarness } from "../gateway/server-methods/board.test-support.js"; +import { resetPluginRuntimeStateForTest } from "../plugins/runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { resolveCanvasDocumentsDir } from "./documents.js"; +import { registerTestWidgetContentKind } from "./widget-tool.content-kinds.test-support.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { createBoardPutCaller } from "./widget-tool.test-support.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetPluginRuntimeStateForTest(); +}); + +const report: BoardReport = { + blocks: [ + { type: "text", title: "Community pulse", text: "Activity is steady." }, + { type: "metrics", items: [{ label: "Authors", value: "312", detail: "This week" }] }, + { type: "table", columns: ["Topic", "Threads"], rows: [["Support", "160"]] }, + { type: "chart", style: "line", points: [{ label: "Mon", value: 12 }] }, + { type: "links", items: [{ label: "Details", url: "https://example.com/details" }] }, + ], +}; + +describe("native report authoring", () => { + it("fails a registered renderer before writing a pinned widget", async () => { + registerTestWidgetContentKind("diagram", () => { + throw new Error("Renderer unavailable"); + }); + const callGateway = vi.fn(); + const tool = createShowWidgetTool({ agentSessionKey: "agent:main:report", callGateway }); + await expect( + tool.execute("renderer-failed", { + title: "Diagram", + kind: "diagram", + pin: true, + widget_code: "diagram:ready", + }), + ).rejects.toThrow("Renderer unavailable"); + expect(callGateway).not.toHaveBeenCalled(); + }); + + it("preserves plugin source kinds named report alongside native report data", async () => { + registerTestWidgetContentKind("report"); + const { mock, callGateway } = createBoardPutCaller(); + const tool = createShowWidgetTool({ + stateDir: tempDirs.make("openclaw-report-plugin-"), + agentSessionKey: "agent:main:report-plugin", + callGateway, + }); + const result = await tool.execute("registered-report", { + title: "Plugin report", + kind: "report", + widget_code: "diagram:ready", + pin: true, + }); + expect(result.details).toMatchObject({ kind: "canvas", view: { id: expect.any(String) } }); + expect(mock).toHaveBeenCalledWith( + "board.widget.put", + expect.objectContaining({ + content: { kind: "registered", contentKind: "report", source: "diagram:ready" }, + }), + ); + }); + + it.each([ + { sessionKey: "agent:main:dashboard:report", agentId: "main" }, + { sessionKey: "global", agentId: "research" }, + ])( + "pins, updates and reopens a report for $agentId/$sessionKey without a document", + async (target) => { + const stateDir = tempDirs.make("openclaw-native-report-"); + const store = createTestBoardStore({ stateDir }); + const sibling = { + sessionKey: "global", + agentId: target.agentId === "main" ? "research" : "main", + }; + const siblingBefore = await store.getSnapshot(sibling); + const { invoke } = createBoardHarness(undefined, {}, store, { + getRuntimeConfig: () => ({ + agents: { ownership: "explicit", entries: { main: {}, research: {} } }, + }), + }); + const callGateway: InProcessGatewayCaller = async ( + method: string, + params: Record, + ) => { + const response = await invoke(method, params); + const [ok, payload, error] = response.mock.calls[0]!; + if (!ok) { + throw new Error(error?.message ?? "Report request failed"); + } + return payload as T; + }; + const tool = createShowWidgetTool({ + stateDir, + agentSessionKey: target.sessionKey, + agentId: target.agentId, + callGateway, + }); + const result = await tool.execute("create-report", { + title: "Community pulse", + name: "community-pulse", + pin: true, + report, + }); + expect(result.details).toMatchObject({ + status: "pinned", + boardWidgetName: "community-pulse", + }); + expect((await store.getSnapshot(target)).widgets[0]).toMatchObject({ + name: "community-pulse", + contentKind: "plugin", + contentOwner: "plugin", + pluginKind: "session:report", + props: report, + grantState: "none", + revision: 1, + }); + const updated = { blocks: [{ type: "text", text: "Refreshed from the agent." }] }; + await createDashboardTool({ + agentSessionKey: target.sessionKey, + agentId: target.agentId, + callGateway, + }).execute("update-report", { + action: "widget_put", + name: "community-pulse", + pluginKind: "session:report", + props: updated, + }); + closeOpenClawAgentDatabasesForTest(); + expect((await store.getSnapshot(target)).widgets[0]).toMatchObject({ + props: updated, + revision: 2, + }); + const view = await invoke("board.get", target); + expect(view.mock.calls[0]?.[0]).toBe(true); + const snapshot = view.mock.calls[0]?.[1] as { widgets: Array> }; + expect(snapshot.widgets[0]).toMatchObject({ pluginKind: "session:report", props: updated }); + expect(snapshot.widgets[0]).not.toHaveProperty("frameUrl"); + expect(snapshot.widgets[0]).not.toHaveProperty("viewTicket"); + expect(await readBoardHtml(store, target, "community-pulse")).toBeUndefined(); + expect(await store.getSnapshot(sibling)).toEqual(siblingBefore); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }, + ); + + it.each([ + { + label: "executable fields", + props: { blocks: [{ type: "text", text: "Hello", script: "ignored" }] }, + }, + { + label: "unsupported blocks", + props: { blocks: [{ type: "image", url: "https://example.com/image.png" }] }, + }, + { + label: "script links", + props: { + blocks: [{ type: "links", items: [{ label: "Invalid", url: "javascript:void(0)" }] }], + }, + }, + { + label: "mismatched table rows", + props: { blocks: [{ type: "table", columns: ["A", "B"], rows: [["one"]] }] }, + }, + { + label: "too many points", + props: { + blocks: [ + { type: "chart", points: Array.from({ length: 41 }, () => ({ label: "A", value: 1 })) }, + ], + }, + }, + { + label: "oversized UTF-8 data", + props: { + blocks: Array.from({ length: 3 }, () => ({ type: "text", text: "é".repeat(2_000) })), + }, + }, + ])("rejects $label at the storage owner without changing the saved report", async ({ props }) => { + const store = createTestBoardStore(); + const target = { sessionKey: "agent:main:report" }; + const content = { kind: "plugin" as const, pluginKind: "session:report", props: report }; + await store.putWidget({ ...target, name: "report", content }); + const before = await store.getSnapshot(target); + await expect( + (async () => + await store.putWidget({ ...target, name: "report", content: { ...content, props } }))(), + ).rejects.toThrow(); + expect(await store.getSnapshot(target)).toEqual(before); + }); + + it.each([ + { pin: false }, + { pin: true, capabilities: { tools: ["prompt"] } }, + { pin: true, kind: "html" }, + { pin: true, widget_code: "

Other content

" }, + { pin: true, presentation: { target: "assistant_message" } }, + { pin: true, presentation: { target: "node_panel" } }, + ])("rejects unsupported report presentation before any write: %j", async (options) => { + const callGateway = vi.fn(); + const tool = createShowWidgetTool({ agentSessionKey: "agent:main:report", callGateway }); + await expect( + tool.execute("invalid-report", { + title: "Report", + report, + ...options, + }), + ).rejects.toThrow("Reports require pin=true"); + expect(callGateway).not.toHaveBeenCalled(); + }); + + it("accepts HTML widget payload when report is an empty object", async () => { + const { mock, callGateway } = createBoardPutCaller(); + const tool = createShowWidgetTool({ + stateDir: tempDirs.make("openclaw-empty-report-html-"), + agentSessionKey: "agent:main:dashboard:empty-report", + callGateway, + }); + const result = await tool.execute("empty-report-with-html", { + title: "SaaS Operations", + pin: true, + report: {}, + widget_code: "
dashboard
", + kind: "html", + capabilities: {}, + presentation: { + target: "assistant_message", + frame: "card", + }, + }); + expect(result.details).toMatchObject({ + kind: "canvas", + view: { boardWidgetName: "saas-operations" }, + }); + expect(mock).toHaveBeenCalledWith( + "board.widget.put", + expect.objectContaining({ + content: expect.objectContaining({ + kind: "html", + html: "
dashboard
", + }), + }), + ); + }); +}); diff --git a/src/canvas/widget-tool.scheduled.test.ts b/src/canvas/widget-tool.scheduled.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..3c180985f007dc99954487bf93e2dc61efed2641 --- /dev/null +++ b/src/canvas/widget-tool.scheduled.test.ts @@ -0,0 +1,78 @@ +// Scheduled show_widget schema and board-only execution coverage. +import { access } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { resolveCanvasDocumentsDir } from "./documents.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { createBoardPutCaller } from "./widget-tool.test-support.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("scheduled show_widget", () => { + it("keeps scheduled widget authoring pinned-only without an inline client", async () => { + const { mock: callGatewayMock, callGateway } = createBoardPutCaller(); + const stateDir = tempDirs.make("openclaw-scheduled-widget-"); + const tool = createShowWidgetTool({ + stateDir, + sessionId: "scheduled-board-only", + agentSessionKey: "agent:main:dashboard:scheduled", + inlineClientAvailable: false, + pinnedOnly: true, + callGateway, + }); + const schema = tool.parameters as { + properties?: { + pin?: { const?: boolean }; + presentation?: { properties?: { target?: unknown } }; + }; + required?: string[]; + }; + + expect(tool.requiredClientCaps).toBeUndefined(); + expect(schema.required).toContain("pin"); + expect(schema.properties?.pin?.const).toBe(true); + expect(schema.properties?.presentation?.properties).not.toHaveProperty("target"); + expect(tool.description).toContain("This surface is pinned-only"); + + await expect( + tool.execute("scheduled-unpinned", { + title: "Scheduled status", + widget_code: "
ready
", + }), + ).rejects.toThrow("pin=true is required for this pinned-only widget surface"); + await expect( + tool.execute("scheduled-target", { + title: "Scheduled status", + widget_code: "
ready
", + pin: true, + presentation: { target: "assistant_message" }, + }), + ).rejects.toThrow("presentation.target is unavailable for this pinned-only widget surface"); + + const result = await tool.execute("scheduled-pinned", { + title: "Scheduled status", + widget_code: "", + name: "scheduled-status", + pin: true, + }); + const text = result.content.find((item) => item.type === "text")?.text; + expect(JSON.parse(text ?? "null")).toEqual({ + status: "pinned", + boardWidgetName: "scheduled-status", + capabilityState: "none", + text: "Widget pinned to dashboard tab main as scheduled-status. Open this dashboard tab in Control UI to view it.", + }); + expect(callGatewayMock).toHaveBeenCalledExactlyOnceWith( + "board.widget.put", + expect.objectContaining({ + sessionKey: "agent:main:dashboard:scheduled", + name: "scheduled-status", + content: { + kind: "html", + html: "", + }, + }), + ); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); +}); diff --git a/src/canvas/widget-tool.script-syntax.test.ts b/src/canvas/widget-tool.script-syntax.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..6835a88e8f7f4c6c597e5fbc028d1c6743f18778 --- /dev/null +++ b/src/canvas/widget-tool.script-syntax.test.ts @@ -0,0 +1,92 @@ +import { access } from "node:fs/promises"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { WidgetHtmlInputError } from "../plugin-sdk/widget-html.js"; +import type { WidgetPresenter } from "../plugins/plugin-registration.types.js"; +import { resetPluginRuntimeStateForTest } from "../plugins/runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { resolveCanvasDocumentsDir } from "./documents.js"; +import { registerTestWidgetContentKind } from "./widget-tool.content-kinds.test-support.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { createBoardPutCaller } from "./widget-tool.test-support.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +beforeEach(() => resetPluginRuntimeStateForTest()); +afterEach(() => { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetPluginRuntimeStateForTest(); +}); + +describe("show_widget script syntax gate", () => { + it.each(["inline", "pinned", "current_channel", "node_panel"] as const)( + "rejects broken scripts before any side effect on %s", + async (route) => { + const stateDir = tempDirs.make("openclaw-widget-syntax-"); + const { mock: gateway, callGateway } = createBoardPutCaller(); + const present = vi.fn(); + const availability = vi.fn(); + const presenters: WidgetPresenter[] = + route === "current_channel" + ? [ + { + target: "current_channel", + description: "Test channel", + capabilities: { sourceKinds: ["html"] }, + match: () => true, + availability, + present, + }, + ] + : [{ target: "node_panel", description: "Test panel", availability, present }]; + const tool = createShowWidgetTool({ + stateDir, + sessionId: "syntax", + agentSessionKey: "agent:main:syntax", + callGateway, + presenters, + }); + const result = tool.execute("broken", { + title: "Broken widget", + widget_code: "

Widget

\n", + pin: route === "pinned", + ...(route === "node_panel" ? { presentation: { target: "node_panel" } } : {}), + }); + await expect(result).rejects.toThrow(WidgetHtmlInputError); + await expect(result).rejects.toThrow( + "widget_code has a JavaScript syntax error in inline script 1 at line 2, column 16: Unterminated string constant. Offending line: ", + }); + const text = result.content.find((item) => item.type === "text")?.text; + expect(JSON.parse(text ?? "null")).toMatchObject({ kind: "canvas" }); + }); + + it("still hosts a valid script as a canvas widget", async () => { + const stateDir = tempDirs.make("openclaw-widget-syntax-"); + const tool = createShowWidgetTool({ stateDir, sessionId: "valid-syntax" }); + const result = await tool.execute("valid", { + title: "Working widget", + widget_code: "", + }); + const text = result.content.find((item) => item.type === "text")?.text; + expect(JSON.parse(text ?? "null")).toMatchObject({ kind: "canvas" }); + await expect(access(resolveCanvasDocumentsDir(stateDir))).resolves.toBeUndefined(); + }); +}); diff --git a/src/canvas/widget-tool.test-support.ts b/src/canvas/widget-tool.test-support.ts new file mode 100644 index 0000000000000000000000000000000000000000..b5ca1ab72004d768ab14ccf07432a37b3dd21168 --- /dev/null +++ b/src/canvas/widget-tool.test-support.ts @@ -0,0 +1,29 @@ +import { vi } from "vitest"; +import type { InProcessGatewayCaller } from "../agents/tools/in-process-gateway.js"; + +export function createBoardPutCaller() { + const mock = vi.fn(async (_method: string, params: Record) => ({ + sessionKey: params.sessionKey, + revision: 1, + tabs: [{ tabId: "main", title: "Main", position: 0, chatDock: "right" }], + widgets: [ + { + name: params.name, + tabId: "main", + contentKind: "plugin", + pluginKind: "diagram:diagram", + sizeW: 6, + sizeH: 4, + position: 0, + grantState: "none", + revision: 1, + }, + ], + resolvedWidgetName: params.name, + })); + const callGateway: InProcessGatewayCaller = async ( + method: string, + params: Record, + ): Promise => (await mock(method, params)) as T; + return { mock, callGateway }; +} diff --git a/src/canvas/widget-tool.test.ts b/src/canvas/widget-tool.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..bbf259e45075672acaa6a1451afbbe9e3dd1caa2 --- /dev/null +++ b/src/canvas/widget-tool.test.ts @@ -0,0 +1,1027 @@ +// Core inline widget validation, materialization, and retention. +import { access, mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { InProcessGatewayCaller } from "../agents/tools/in-process-gateway.js"; +import { readBoardHtml, createTestBoardStore } from "../boards/board-store.test-support.js"; +import { retainLegacyDefaultAgentId } from "../config/legacy.default-agent-owner.js"; +import type { OpenClawConfig } from "../config/types.js"; +import { createBoardHandlers } from "../gateway/server-methods/board.js"; +import type { GatewayRequestContext, RespondFn } from "../gateway/server-methods/types.js"; +import type { WidgetPresenter } from "../plugins/plugin-registration.types.js"; +import { createEmptyPluginRegistry } from "../plugins/registry-empty.js"; +import { resetPluginRuntimeStateForTest, setActivePluginRegistry } from "../plugins/runtime.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { resolveCanvasDocumentsDir } from "./documents.js"; +import { registerTestWidgetContentKind as registerDiagramContentKind } from "./widget-tool.content-kinds.test-support.js"; +import { createShowWidgetTool } from "./widget-tool.js"; +import { createBoardPutCaller } from "./widget-tool.test-support.js"; +import { buildWidgetDocument } from "./wrap.js"; + +const WIDGET_CODE_MAX_CHARS = 262_144; +const PINNED_WIDGET_MAX_UTF8_BYTES = 256 * 1024; +const WIDGET_MAX_PER_SCOPE = 32; +const tempDirs: string[] = []; + +beforeEach(() => { + resetPluginRuntimeStateForTest(); +}); + +afterEach(async () => { + vi.useRealTimers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + resetPluginRuntimeStateForTest(); + await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); +}); + +async function createStateDir(): Promise { + const stateDir = await mkdtemp(path.join(tmpdir(), "openclaw-widget-tool-")); + tempDirs.push(stateDir); + return stateDir; +} + +function createLiveBoardTestContext( + broadcast: ReturnType = vi.fn(), + cfg: OpenClawConfig = { agents: { entries: { main: {} } } }, +): GatewayRequestContext { + const context = { + broadcast, + getSessionEventSubscriberConnIds: () => new Set(), + getRuntimeConfig: () => cfg, + } as unknown as GatewayRequestContext; + context.resolveGatewayContext = () => context; + return context; +} + +function resolveCanvasDocumentDir(stateDir: string, documentId: string): string { + return path.join(resolveCanvasDocumentsDir(stateDir), documentId); +} + +async function executeWidget(params: { + stateDir: string; + sessionId?: string; + title?: string; + widgetCode: string; + agentId?: string; + agentSessionKey?: string; + callGateway?: InProcessGatewayCaller; + pin?: boolean; + name?: string; + tab?: string; + size?: "sm" | "md" | "lg" | "xl" | "full"; + presentation?: { + target?: "assistant_message" | "node_panel"; + frame?: "card" | "full-bleed" | "frameless"; + }; + presenters?: readonly WidgetPresenter[]; + after?: string; + capabilities?: { netOrigins?: string[]; tools?: string[] }; + kind?: string; +}) { + const tool = createShowWidgetTool({ + stateDir: params.stateDir, + sessionId: params.sessionId ?? "widget-session", + agentId: params.agentId ?? "main", + agentSessionKey: params.agentSessionKey, + callGateway: params.callGateway, + presenters: params.presenters, + }); + const result = await tool.execute("widget-call", { + title: params.title ?? "Widget title", + widget_code: params.widgetCode, + ...(params.pin !== undefined ? { pin: params.pin } : {}), + ...(params.name ? { name: params.name } : {}), + ...(params.tab ? { tab: params.tab } : {}), + ...(params.size ? { size: params.size } : {}), + ...(params.presentation ? { presentation: params.presentation } : {}), + ...(params.after ? { after: params.after } : {}), + ...(params.capabilities ? { capabilities: params.capabilities } : {}), + ...(params.kind ? { kind: params.kind } : {}), + }); + const text = result.content.find((item) => item.type === "text")?.text; + if (!text) { + throw new Error("expected widget tool text result"); + } + const parsed = JSON.parse(text) as { + kind?: string; + presentation?: { target?: string; title?: string; sandbox?: string }; + view?: { id?: string; url?: string; boardWidgetName?: string }; + text?: string; + }; + const viewId = parsed.view?.id; + const url = parsed.view?.url; + if (parsed.kind !== "canvas" || !viewId || !url) { + throw new Error("expected canvas preview handle"); + } + return { + viewId, + url, + sandbox: parsed.presentation?.sandbox, + resultText: parsed.text, + boardWidgetName: parsed.view?.boardWidgetName, + target: parsed.presentation?.target, + text, + }; +} + +describe("show_widget", () => { + it("builds a sorted kind enum and routes registered source through board put", async () => { + registerDiagramContentKind(); + const stateDir = await createStateDir(); + const { mock: callGatewayMock, callGateway } = createBoardPutCaller(); + const tool = createShowWidgetTool({ + stateDir, + sessionId: "registered", + agentSessionKey: "agent:main:registered", + callGateway, + }); + const kindSchema = (tool.parameters as { properties?: { kind?: { enum?: string[] } } }) + .properties?.kind; + + expect(kindSchema?.enum).toEqual(["html", "diagram"]); + await tool.execute("registered", { + title: "Diagram", + widget_code: "diagram:ready", + kind: "diagram", + pin: true, + }); + expect(callGatewayMock).toHaveBeenCalledWith( + "board.widget.put", + expect.objectContaining({ + content: { kind: "registered", contentKind: "diagram", source: "diagram:ready" }, + }), + ); + + setActivePluginRegistry(createEmptyPluginRegistry()); + await expect( + tool.execute("stale", { + title: "Diagram", + widget_code: "diagram:ready", + kind: "diagram", + }), + ).rejects.toThrow( + 'widget kind "diagram" is unavailable; enable the plugin that provides it and retry', + ); + }); + + it("uses board-only delivery when inline hosting is disabled", async () => { + registerDiagramContentKind(); + const stateDir = await createStateDir(); + const { mock: callGatewayMock, callGateway } = createBoardPutCaller(); + const tool = createShowWidgetTool({ + stateDir, + sessionId: "board-only", + agentSessionKey: "agent:main:board-only", + inlineHostEnabled: false, + callGateway, + }); + + expect(tool.description).toContain( + "Inline previews are unavailable this turn; set pin=true to save to the session dashboard", + ); + expect(tool.description).not.toContain("Keep one-off visualizations inline"); + await expect( + tool.execute("unpinned", { + title: "Diagram", + widget_code: "diagram:ready", + kind: "diagram", + }), + ).rejects.toThrow( + "inline widget hosting is disabled; set pin=true to place the widget on the session dashboard", + ); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + + const result = await tool.execute("pinned", { + title: "Diagram", + widget_code: "diagram:ready", + kind: "diagram", + pin: true, + }); + const text = result.content.find((item) => item.type === "text")?.text; + expect(JSON.parse(text ?? "null")).toEqual({ + status: "pinned", + boardWidgetName: "diagram", + capabilityState: "none", + text: "Widget pinned to dashboard tab main as diagram. Open this dashboard tab in Control UI to view it.", + }); + expect(callGatewayMock).toHaveBeenCalledExactlyOnceWith( + "board.widget.put", + expect.objectContaining({ + content: { kind: "registered", contentKind: "diagram", source: "diagram:ready" }, + }), + ); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); + + it("keeps widget documents from duplicating host-owned metadata and controls", () => { + const description = createShowWidgetTool().description; + + expect(description).toContain("openclaw.host.controlUiBaseUrl"); + expect(description).toContain("read it at click time"); + expect(description).toContain('target="_blank" and rel="noopener noreferrer"'); + expect(description).toContain("`title` is host metadata"); + expect(description).toContain("Start directly with content"); + expect(description).toContain("do not repeat the title"); + }); + + it("builds provider-safe kind and presentation enums from both registries", () => { + registerDiagramContentKind(); + const tool = createShowWidgetTool(); + const properties = ( + tool.parameters as { + properties?: Record< + string, + { + anyOf?: unknown; + enum?: string[]; + properties?: Record; + } + >; + } + ).properties; + expect(properties?.kind).toMatchObject({ enum: ["html", "diagram"] }); + expect(properties?.size).toMatchObject({ enum: ["sm", "md", "lg", "xl", "full"] }); + expect(properties?.presentation?.properties?.target).toMatchObject({ + enum: ["assistant_message"], + }); + expect(properties?.presentation?.properties?.target?.description).not.toContain("node_panel"); + expect(properties?.presentation?.properties?.frame).toMatchObject({ + enum: ["card", "full-bleed", "frameless"], + }); + expect(properties?.size?.anyOf).toBeUndefined(); + expect(properties?.presentation?.properties?.target?.anyOf).toBeUndefined(); + expect(properties?.presentation?.properties?.frame?.anyOf).toBeUndefined(); + expect(tool.description).toContain("registered kinds are diagram"); + + const presenter: WidgetPresenter = { + target: "node_panel", + description: "Show on a connected device panel", + availability: async () => ({ ok: true, value: { available: true } }), + present: async () => ({ + ok: true, + value: { kind: "node", nodeId: "mac-panel", nodeName: "Studio" }, + }), + }; + const withPresenterTool = createShowWidgetTool({ presenters: [presenter] }); + const withPresenter = withPresenterTool.parameters as { + properties?: Record< + string, + { + enum?: string[]; + properties?: Record; + } + >; + }; + expect(withPresenter.properties?.kind).toMatchObject({ enum: ["html", "diagram"] }); + expect(withPresenter.properties?.presentation?.properties?.target).toMatchObject({ + enum: ["assistant_message", "node_panel"], + description: expect.stringContaining("node_panel: Show on a connected device panel"), + }); + expect(withPresenterTool.description).toContain("registered kinds are diagram"); + expect(withPresenterTool.description).toContain( + "Use presentation.target to choose a registered device surface.", + ); + }); + + it("routes node-panel presentation and reports the selected device", async () => { + const stateDir = await createStateDir(); + const availability = vi.fn(async () => ({ + ok: true as const, + value: { available: true as const }, + })); + const present = vi.fn(async () => ({ + ok: true as const, + value: { kind: "node" as const, nodeId: "mac-panel", nodeName: "Studio" }, + })); + const result = await executeWidget({ + stateDir, + title: "Status", + widgetCode: "

ready

", + agentSessionKey: "agent:main:status", + presentation: { target: "node_panel" }, + presenters: [ + { + target: "node_panel", + description: "Show on a connected device panel", + availability, + present, + }, + ], + }); + + expect(result.target).toBe("node_panel"); + expect(result.resultText).toContain("presented on Studio (mac-panel)"); + expect(availability).toHaveBeenCalledWith({ sessionKey: "agent:main:status" }); + expect(present).toHaveBeenCalledWith({ + document: { + kind: "html", + html: expect.stringContaining("

ready

"), + hostedUrl: result.url, + }, + title: "Status", + context: { sessionKey: "agent:main:status" }, + }); + }); + + it.each([ + { + name: "has no registered presenter", + presenters: [] as WidgetPresenter[], + expected: "No widget presenter is registered", + }, + { + name: "has no eligible node", + presenters: [ + { + target: "node_panel" as const, + description: "Show on a connected device panel", + availability: async () => ({ + ok: false as const, + error: { code: "no_eligible_node" as const, message: "No connected device." }, + }), + present: async () => { + throw new Error("present must not run"); + }, + }, + ], + expected: "No connected device.", + }, + { + name: "hits a node error", + presenters: [ + { + target: "node_panel" as const, + description: "Show on a connected device panel", + availability: async () => ({ ok: true as const, value: { available: true as const } }), + present: async () => ({ + ok: false as const, + error: { + code: "node_error" as const, + message: "Canvas is disabled.", + nodeId: "mac-panel", + }, + }), + }, + ], + expected: "Canvas is disabled.", + }, + ])( + "falls back inline with an actionable message when node presentation $name", + async ({ presenters, expected }) => { + const stateDir = await createStateDir(); + const result = await executeWidget({ + stateDir, + widgetCode: "

inline fallback

", + presentation: { target: "node_panel" }, + presenters, + }); + + expect(result.target).toBe("assistant_message"); + expect(result.resultText).toContain(expected); + expect(result.resultText).toContain("available inline here"); + expect(result.resultText).toMatch( + /Pair a canvas-capable device|Retry the requested presentation destination/u, + ); + await expect( + access(resolveCanvasDocumentDir(stateDir, result.viewId)), + ).resolves.toBeUndefined(); + }, + ); + + it("enforces current-presenter source kinds and byte limits in core", async () => { + registerDiagramContentKind(); + const presenter: WidgetPresenter = { + target: "current_channel", + description: "HTML-only current channel", + capabilities: { sourceKinds: ["html"], maxSourceBytes: 8 }, + match: () => true, + availability: async () => ({ ok: true, value: { available: true } }), + present: async () => { + throw new Error("present must not run"); + }, + }; + const tool = createShowWidgetTool({ + inlineClientAvailable: false, + presenters: [presenter], + presenterContext: {}, + }); + const kindSchema = (tool.parameters as { properties?: { kind?: { enum?: string[] } } }) + .properties?.kind; + + expect(kindSchema?.enum).toEqual(["html"]); + expect(tool.description).not.toContain("registered kinds are diagram"); + await expect( + tool.execute("oversized-current", { title: "Large", widget_code: "123456789" }), + ).rejects.toThrow("widget_code exceeds maximum size (8 bytes)"); + await expect( + tool.execute("unsupported-current", { + title: "Diagram", + widget_code: "diagram:ready", + kind: "diagram", + }), + ).rejects.toThrow("inline widget hosting is disabled"); + }); + + it("fails visibly without inline fallback and uses a real inline route when available", async () => { + const presenter: WidgetPresenter = { + target: "current_channel", + description: "Failing current channel", + capabilities: { sourceKinds: ["html"] }, + match: () => true, + availability: async () => ({ ok: true, value: { available: true } }), + present: async () => ({ + ok: false, + error: { code: "presentation_error", message: "delivery rejected" }, + }), + }; + const noInline = createShowWidgetTool({ + inlineClientAvailable: false, + presenters: [presenter], + presenterContext: {}, + }); + await expect( + noInline.execute("no-inline", { title: "Status", widget_code: "

ready

" }), + ).rejects.toThrow("Widget presentation failed: delivery rejected"); + + const stateDir = await createStateDir(); + const withInline = createShowWidgetTool({ + stateDir, + sessionId: "inline-fallback", + inlineClientAvailable: true, + presenters: [presenter], + presenterContext: {}, + }); + const fallback = await withInline.execute("with-inline", { + title: "Status", + widget_code: "

ready

", + }); + const parsed = JSON.parse( + fallback.content[0]?.type === "text" ? fallback.content[0].text : "null", + ); + expect(parsed).toMatchObject({ + kind: "canvas", + presentation: { target: "assistant_message" }, + text: expect.stringContaining("delivery rejected. The widget is available inline here."), + }); + }); + + it("reports pin success and presentation failure as an explicit partial outcome", async () => { + const { callGateway } = createBoardPutCaller(); + const presenter: WidgetPresenter = { + target: "current_channel", + description: "Failing current channel", + capabilities: { sourceKinds: ["html"] }, + match: () => true, + availability: async () => ({ ok: true, value: { available: true } }), + present: async () => ({ + ok: false, + error: { code: "presentation_error", message: "delivery rejected" }, + }), + }; + const tool = createShowWidgetTool({ + agentSessionKey: "agent:main:partial", + callGateway, + inlineClientAvailable: false, + presenters: [presenter], + presenterContext: {}, + }); + const result = await tool.execute("partial", { + title: "Status", + widget_code: "

ready

", + pin: true, + }); + const parsed = JSON.parse(result.content[0]?.type === "text" ? result.content[0].text : "null"); + + expect(parsed).toMatchObject({ + status: "partial", + boardWidgetName: "status", + presentation: { + target: "current_channel", + status: "failed", + error: { code: "presentation_error", message: "delivery rejected" }, + }, + text: expect.stringContaining( + "pinned to dashboard tab main as status, but presentation failed", + ), + }); + }); + + it("rejects empty and oversized widget code", async () => { + const stateDir = await createStateDir(); + const tool = createShowWidgetTool({ stateDir, sessionId: "validation" }); + + await expect(tool.execute("empty", { title: "Empty", widget_code: " " })).rejects.toThrow( + "widget_code required", + ); + await expect( + tool.execute("oversized", { + title: "Too large", + widget_code: "x".repeat(WIDGET_CODE_MAX_CHARS + 1), + }), + ).rejects.toThrow(`widget_code exceeds maximum size (${WIDGET_CODE_MAX_CHARS} characters)`); + }); + + it("rejects pinning without a session before creating a Canvas document", async () => { + const stateDir = await createStateDir(); + const tool = createShowWidgetTool({ stateDir, sessionId: "missing-agent-session" }); + + await expect( + tool.execute("pin", { + title: "Pinned", + widget_code: "

never materialized

", + pin: true, + }), + ).rejects.toThrow("pin requires an agent session"); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); + + it("rejects multibyte pin input that exceeds the wrapped UTF-8 budget", async () => { + const stateDir = await createStateDir(); + const callGateway = vi.fn(); + const title = "Multibyte"; + const wrapperBytes = Buffer.byteLength(buildWidgetDocument(title, ""), "utf8"); + const widgetCode = "é".repeat( + Math.floor((PINNED_WIDGET_MAX_UTF8_BYTES - wrapperBytes) / 2) + 1, + ); + expect(widgetCode.length).toBeLessThan(WIDGET_CODE_MAX_CHARS); + expect(Buffer.byteLength(buildWidgetDocument(title, widgetCode), "utf8")).toBeGreaterThan( + PINNED_WIDGET_MAX_UTF8_BYTES, + ); + const tool = createShowWidgetTool({ + stateDir, + sessionId: "wrapped-budget", + agentSessionKey: "agent:main:wrapped-budget", + callGateway, + }); + + await expect( + tool.execute("pin", { title, widget_code: widgetCode, pin: true }), + ).rejects.toThrow( + `pin exceeds effective dashboard budget (${PINNED_WIDGET_MAX_UTF8_BYTES} UTF-8 bytes after wrapping)`, + ); + expect(callGateway).not.toHaveBeenCalled(); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); + + it("does not create an inline Canvas document when dashboard pinning fails", async () => { + const stateDir = await createStateDir(); + const callGateway = vi.fn(async () => { + throw new Error("board tab not found: missing"); + }); + + await expect( + executeWidget({ + stateDir, + agentSessionKey: "agent:main:failed-pin", + widgetCode: "

never materialized

", + pin: true, + tab: "missing", + callGateway, + }), + ).rejects.toThrow("board tab not found: missing"); + expect(callGateway).toHaveBeenCalledOnce(); + await expect(access(resolveCanvasDocumentsDir(stateDir))).rejects.toThrow(); + }); + + it("requires dashboard pinning for declared capabilities", async () => { + const stateDir = await createStateDir(); + const tool = createShowWidgetTool({ stateDir, sessionId: "capabilities" }); + + await expect( + tool.execute("capabilities", { + title: "Weather", + widget_code: "

weather

", + capabilities: { netOrigins: ["https://api.open-meteo.com"] }, + }), + ).rejects.toThrow("capabilities require pin=true"); + }); + + it("wraps SVG widgets with the stable result and sandbox contracts", async () => { + const stateDir = await createStateDir(); + const { viewId, url, sandbox, text } = await executeWidget({ + stateDir, + title: "", + widgetCode: ' ', + }); + + expect(viewId).toMatch(/^cv_[a-f0-9]{32}$/); + expect(url).toBe(`/__openclaw__/canvas/documents/${viewId}/index.html`); + expect(JSON.parse(text)).toMatchObject({ + kind: "canvas", + presentation: { target: "assistant_message", title: "", sandbox: "scripts" }, + text: `Widget hosted at ${url}`, + }); + expect(sandbox).toBe("scripts"); + const html = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, viewId), "index.html"), + "utf8", + ); + expect(html).toContain( + `Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline' https://cdnjs.cloudflare.com`, + ); + expect(html).toContain("font-src data:"); + expect(html).toContain("img-src data:; connect-src 'none'"); + expect(html).toContain("<Status>"); + expect(html).toContain("--accent:#bd4531"); + expect(html).toContain("--accent:#ff5c5c"); + expect(html).toContain("--accent-fill:#d13c3c"); + expect(html).toContain('", + }); + const html = await readFile( + path.join(resolveCanvasDocumentDir(stateDir, viewId), "index.html"), + "utf8", + ); + + expect(html).not.toContain(''); + expect(html.indexOf("window.sendPrompt")).toBeLessThan(html.indexOf("
")); + expect(html).toContain("openclaw:widget-theme"); + expect(html.indexOf("openclaw:widget-theme")).toBeLessThan(html.indexOf("
")); + expect(html).toContain("openclaw:widget-snapshot-request"); + expect(html.indexOf("openclaw:widget-theme")).toBeLessThan( + html.indexOf("openclaw:widget-snapshot-request"), + ); + expect(html.indexOf("openclaw:widget-snapshot-request")).toBeLessThan( + html.indexOf("
"), + ); + expect(html).toContain("openclaw:widget-prompt-offer"); + expect(html).toContain("openclaw:widget-bridge-port-offer"); + expect(html).toContain("openclaw:widget-bridge-request"); + expect(html).toContain("prompt:freeze({send:sendPrompt})"); + expect(html).toContain('state:freeze({emit:payload=>request("state.emit"'); + expect(html).toContain('data:freeze({read:(bindingId,params)=>request("data.read"'); + expect(html).toContain('action:freeze({run:(action,params)=>request("action.run"'); + expect(html).toContain('cron:freeze({trigger:jobId=>request("cron.trigger"'); + expect(html).toContain("navigator.userActivation"); + expect(html).toContain("c.port1.postMessage.bind(c.port1)"); + expect(html).toContain("b.port1.postMessage.bind(b.port1)"); + expect(html).toContain('bridgePost({type:"openclaw:widget-bridge-request"'); + expect(html).not.toContain( + 'post({type:"openclaw:widget-bridge-request",id,method,params,ticket},"*")', + ); + expect(html).toContain('promptPost({type:"openclaw:widget-prompt"'); + expect(html).not.toContain('window.parent.postMessage({type:"openclaw:widget-prompt",'); + expect(html).toContain("const post=(message,origin)=>parent.postMessage(message,origin)"); + expect(html).toContain('query.call(root,"script")'); + expect(html).toContain('queryDocument("canvas")'); + expect(html).toContain("canvasWidth*canvasHeight>16777216"); + expect(html).toContain('toDataURL.call(canvas,"image/png")'); + }); + + it("uses opaque ids and evicts the oldest widget within a session scope", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-07-07T00:00:00.000Z")); + const stateDir = await createStateDir(); + const first = await executeWidget({ stateDir, widgetCode: "

0

" }); + for (let index = 1; index <= WIDGET_MAX_PER_SCOPE; index += 1) { + vi.setSystemTime(new Date(`2026-07-07T00:00:${String(index).padStart(2, "0")}.000Z`)); + await executeWidget({ stateDir, widgetCode: `

${index}

` }); + } + + await expect(access(resolveCanvasDocumentDir(stateDir, first.viewId))).rejects.toThrow(); + const entries = await readdir(path.join(stateDir, "canvas", "documents")); + expect(entries).toHaveLength(WIDGET_MAX_PER_SCOPE); + }); +}); diff --git a/src/canvas/widget-tool.ts b/src/canvas/widget-tool.ts new file mode 100644 index 0000000000000000000000000000000000000000..8426a073976a4894c12ce759ed5fd04f660f55f6 --- /dev/null +++ b/src/canvas/widget-tool.ts @@ -0,0 +1,640 @@ +/** Agent-facing inline chat widget tool. */ +import { createHash } from "node:crypto"; +import { truncateCodePoints } from "@openclaw/normalization-core/code-points"; +import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce"; +import { Type } from "typebox"; +import type { BoardWidgetPutResult } from "../../packages/gateway-protocol/src/index.js"; +import { optionalStringEnum } from "../agents/schema/string-enum.js"; +import { type AnyAgentTool, jsonResult, readToolStringParam } from "../agents/tools/common.js"; +import { + callInProcessGatewayTool, + type InProcessGatewayCaller, +} from "../agents/tools/in-process-gateway.js"; +import { normalizeBoardWidgetDeclared } from "../boards/board-capabilities.js"; +import { + BOARD_REPORT_GUIDANCE, + BOARD_REPORT_WIDGET_KIND, + parseBoardReport, +} from "../boards/board-report.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { + assertWidgetHtmlSize, + WIDGET_CDN_ORIGINS, + WidgetHtmlInputError, +} from "../plugin-sdk/widget-html.js"; +import { + listBoardWidgetContentKinds, + resolveBoardWidgetContentKind, +} from "../plugins/board-widget-content-kinds.js"; +import { describeDashboardCapabilities } from "../plugins/dashboard-capabilities.js"; +import type { + WidgetPresentationError, + WidgetPresentationSuccess, + WidgetPresenter, + WidgetPresenterContext, + WidgetPresenterDocument, +} from "../plugins/plugin-registration.types.js"; +import { getActivePluginRegistry } from "../plugins/runtime.js"; +import { getPluginRuntimeGatewayRequestScope } from "../plugins/runtime/gateway-request-scope.js"; +import { createCanvasDocument } from "./documents.js"; +import { findWidgetScriptSyntaxError } from "./widget-script-syntax.js"; +import { buildWidgetDocument } from "./wrap.js"; + +const SHOW_WIDGET_REQUIRED_CLIENT_CAPS = ["inline-widgets"]; +const WIDGET_CODE_MAX_CHARS = 262_144; +const PINNED_WIDGET_MAX_UTF8_BYTES = 256 * 1024; +const WIDGET_MAX_PER_SCOPE = 32; + +function currentPluginRegistry() { + return getPluginRuntimeGatewayRequestScope()?.pluginRegistry ?? getActivePluginRegistry(); +} + +export function hasRegisteredShowWidgetKinds(): boolean { + return listBoardWidgetContentKinds(currentPluginRegistry()).length > 0; +} + +function createShowWidgetToolSchema( + kinds: readonly string[], + presenters: readonly WidgetPresenter[], + capabilityGuidance: string, + pinnedOnly: boolean, + reportAvailable: boolean, +) { + const presenterTargets = presenters.flatMap((presenter) => + presenter.target === "current_channel" ? [] : [presenter.target], + ); + const targets = ["assistant_message", ...presenterTargets] as const; + const presenterDescriptions = presenters.flatMap((presenter) => + presenter.target === "current_channel" ? [] : [`${presenter.target}: ${presenter.description}`], + ); + const widgetCode = Type.String({ + description: + "Required for HTML/SVG or registered source. For HTML, send a fragment with optional style/script tags, not a full document or file path. Use fluid widths and wrap or stack narrow layouts; reserve horizontal scrolling for exact geometry.", + }); + return Type.Object({ + title: Type.String({ description: "Short host title; do not repeat it inside the widget" }), + widget_code: reportAvailable ? Type.Optional(widgetCode) : widgetCode, + ...(reportAvailable + ? { + report: Type.Optional( + Type.Record(Type.String(), Type.Unknown(), { + description: `Native dashboard data; requires pin=true. Omit widget_code, kind, capabilities, and presentation.target. ${BOARD_REPORT_GUIDANCE}`, + }), + ), + } + : {}), + kind: optionalStringEnum(kinds, { + description: `Widget source kind: ${kinds.join(", ")}`, + }), + name: Type.Optional( + Type.String({ + pattern: "^[a-z0-9][a-z0-9._-]{0,63}$", + description: + "Stable dashboard widget name; reuse the same name with pin=true and new report data or widget_code to update", + }), + ), + pin: pinnedOnly + ? Type.Literal(true, { + description: "Required: this surface can only author pinned widgets", + }) + : Type.Optional( + Type.Boolean({ + description: + "Pin only for an explicit dashboard request or multiple non-code visualizations", + }), + ), + tab: Type.Optional( + Type.String({ pattern: "^[a-z0-9-]{1,40}$", description: "Dashboard tab slug" }), + ), + size: optionalStringEnum(["sm", "md", "lg", "xl", "full"] as const, { + description: "Dashboard size: sm, md, lg, xl, or full", + }), + presentation: Type.Optional( + Type.Object({ + ...(pinnedOnly + ? {} + : { + target: optionalStringEnum(targets, { + description: [ + "Where to show the widget. assistant_message: inline in chat", + ...presenterDescriptions, + ].join("; "), + }), + }), + frame: optionalStringEnum(["card", "full-bleed", "frameless"] as const, { + description: "Pinned dashboard frame: card, full-bleed, or frameless", + }), + }), + ), + after: Type.Optional( + Type.String({ + pattern: "^[a-z0-9][a-z0-9._-]{0,63}$", + description: "Place after this dashboard widget name", + }), + ), + capabilities: Type.Optional( + Type.Object({ + netOrigins: Type.Optional( + Type.Array(Type.String(), { + description: "Exact HTTPS origins the pinned widget may fetch after approval", + }), + ), + tools: Type.Optional( + Type.Array(Type.String(), { + description: `Pinned widget host tools: prompt or cron.trigger:; grant each read/action ID below unless a scoped grant is specified. ${capabilityGuidance}`, + }), + ), + }), + ), + }); +} + +type ShowWidgetToolOptions = { + sessionId?: string; + agentId?: string; + agentSessionKey?: string; + stateDir?: string; + callGateway?: InProcessGatewayCaller; + inlineHostEnabled?: boolean; + inlineClientAvailable?: boolean; + /** Admitted callers without a rendering client may author only durable dashboard widgets. */ + pinnedOnly?: boolean; + presenters?: readonly WidgetPresenter[]; + presenterContext?: WidgetPresenterContext; +}; + +type WidgetPresentationAttempt = + | { ok: true; value: WidgetPresentationSuccess } + | { ok: false; error: WidgetPresentationError }; + +async function presentWidget(params: { + presenter?: WidgetPresenter; + document: WidgetPresenterDocument; + title: string; + context: WidgetPresenterContext; +}): Promise { + const presenter = params.presenter; + if (!presenter) { + return { + ok: false, + error: { + code: "no_eligible_node", + message: "No widget presenter is registered for this target.", + }, + }; + } + const errorCode = presenter.target === "current_channel" ? "presentation_error" : "node_error"; + try { + const availability = await presenter.availability(params.context); + if (!availability.ok) { + return availability; + } + return await presenter.present({ + document: params.document, + title: params.title, + context: params.context, + }); + } catch (error) { + return { + ok: false, + error: { code: errorCode, message: formatErrorMessage(error) }, + }; + } +} + +export function resolveCurrentChannelWidgetPresenter( + presenters: readonly WidgetPresenter[], + context: WidgetPresenterContext, +): Extract | undefined { + const matches = presenters.filter( + (presenter): presenter is Extract => { + if (presenter.target !== "current_channel") { + return false; + } + try { + return presenter.match(context); + } catch { + return false; + } + }, + ); + return matches.length === 1 ? matches[0] : undefined; +} + +function widgetPresentationFailureText( + error: WidgetPresentationError, + inlineAvailable: boolean, +): string { + const message = /[.!?]$/u.test(error.message) ? error.message : `${error.message}.`; + if (!inlineAvailable) { + return message; + } + const nextStep = + error.code === "no_eligible_node" + ? "Pair a canvas-capable device or open the OpenClaw app, then retry." + : "Retry the requested presentation destination when it is available."; + return `${message} The widget is available inline here. ${nextStep}`; +} + +function slugWidgetName(title: string): string { + const slug = title + .normalize("NFKD") + .replace(/[\u0300-\u036f]/gu, "") + .toLowerCase() + .replace(/[^a-z0-9]+/gu, "-") + .replace(/^-+|-+$/gu, ""); + if (slug && slug.length <= 64) { + return slug; + } + const suffix = createHash("sha256").update(title).digest("hex").slice(0, 8); + const prefix = (slug || "widget").slice(0, 55).replace(/-+$/gu, "") || "widget"; + return `${prefix}-${suffix}`; +} + +function generatedWidgetIdentity(title: string, preferredName: string) { + const key = createHash("sha256").update(title.trim().normalize("NFC")).digest("hex"); + const prefix = preferredName.slice(0, 55).replace(/-+$/gu, "") || "widget"; + return { + source: "show_widget" as const, + key, + fallbackName: `${prefix}-${key.slice(0, 8)}`, + }; +} + +function boardWidgetTitle(title: string): string | undefined { + const normalized = title.trim(); + return normalized ? truncateCodePoints(normalized, 80) : undefined; +} + +function resolveRetentionScope(options: ShowWidgetToolOptions): string { + const scope = options.sessionId + ? `session:${options.sessionId}` + : `agent:${options.agentId ?? "default"}`; + return createHash("sha256").update(scope).digest("hex"); +} + +function assertPinnedWidgetDocumentSize(html: string): void { + if (Buffer.byteLength(html, "utf8") > PINNED_WIDGET_MAX_UTF8_BYTES) { + throw new WidgetHtmlInputError( + `pin exceeds effective dashboard budget (${PINNED_WIDGET_MAX_UTF8_BYTES} UTF-8 bytes after wrapping)`, + ); + } +} + +/** Creates a self-contained widget hosted by OpenClaw core. */ +export function createShowWidgetTool(options: ShowWidgetToolOptions = {}): AnyAgentTool { + const gatewayCall = options.callGateway ?? callInProcessGatewayTool; + const pinnedOnly = options.pinnedOnly === true; + const inlineHostEnabled = options.inlineHostEnabled !== false; + const inlineAvailable = + !pinnedOnly && inlineHostEnabled && options.inlineClientAvailable !== false; + const registeredKinds = listBoardWidgetContentKinds(currentPluginRegistry()); + const allKinds = ["html", ...registeredKinds]; + const presenters = options.presenters ?? []; + const presenterContext = + options.presenterContext ?? + (options.agentSessionKey ? { sessionKey: options.agentSessionKey } : {}); + const currentChannelPresenter = pinnedOnly + ? undefined + : resolveCurrentChannelWidgetPresenter(presenters, presenterContext); + const kinds = + currentChannelPresenter && !inlineAvailable + ? allKinds.filter((kind) => currentChannelPresenter.capabilities.sourceKinds.includes(kind)) + : allKinds; + const advertisedRegisteredKinds = kinds.filter((kind) => kind !== "html"); + const reportAvailable = Boolean(options.agentSessionKey?.trim()); + const reportGuidance = reportAvailable + ? " Prefer the report argument with pin=true for data reports; these render natively on the dashboard without a document frame. Reports are dashboard-only; omit widget_code, kind, capabilities, and presentation.target." + : ""; + const explicitPresenters = pinnedOnly + ? [] + : presenters.filter((presenter) => presenter.target !== "current_channel"); + const presenterPrompt = + explicitPresenters.length > 0 + ? " Use presentation.target to choose a registered device surface." + : ""; + const usageGuidance = pinnedOnly + ? "This surface is pinned-only: set pin=true to create or update a durable session dashboard widget. Inline and device presentation are unavailable." + : currentChannelPresenter + ? "Show widgets through the current channel presenter; follow result.presentation for delivery." + : inlineAvailable + ? "Keep one-off visualizations inline; pin for explicit dashboard requests or multiple non-code visualizations." + : "Inline previews are unavailable this turn; set pin=true to save to the session dashboard."; + return { + label: "Show Widget", + name: "show_widget", + description: `Visual helps? Make widget. Do not wait for ask. ${usageGuidance} Update pinned HTML by name. Use for code architecture, execution traces, performance comparisons, interactive explanations, UI mockups, and dashboards. Text clearer? Skip. Load the visualize skill when available for composition and dashboard authoring. The source kind defaults to html${advertisedRegisteredKinds.length ? ` and registered kinds are ${advertisedRegisteredKinds.join(", ")}` : ""}. Send markup directly in widget_code. Scripts, stylesheets, and fonts may load from ${WIDGET_CDN_ORIGINS.join(", ")}; pin library versions. Inline widgets cannot fetch APIs. Pinned data access needs declared and granted capabilities.netOrigins or capabilities.tools; inline previews never inherit those grants. Keep filters and controls local; user-clicked openclaw.prompt.send(text) requests an agent follow-up in the Control UI. Data, action, state, and cron host APIs are dashboard-only. openclaw.host.controlUiBaseUrl is the Control UI origin plus base path after dashboard initialization, otherwise null; read it at click time. Dashboard HTML links support HTTP(S) destinations only; open them with target="_blank" and rel="noopener noreferrer". Put local workspace file links in chat Markdown, not widget HTML; file:// links cannot open the Files panel. \`title\` is host metadata. Start directly with content; do not repeat the title or recreate dashboard chrome. Use host theme variables such as --text, --muted, --card, --border, --accent, --font-body, and --font-mono. Inline script syntax errors return line and column; fix and retry. Check library loading and rendered interactions; hosting success alone is not visual proof.${reportGuidance}${presenterPrompt}`, + parameters: createShowWidgetToolSchema( + kinds, + explicitPresenters, + describeDashboardCapabilities(currentPluginRegistry()), + pinnedOnly, + reportAvailable, + ), + ...(currentChannelPresenter || pinnedOnly + ? {} + : { requiredClientCaps: SHOW_WIDGET_REQUIRED_CLIENT_CAPS }), + execute: async (_toolCallId, args) => { + const params = args as Record; + const requestedKind = readToolStringParam(params, "kind"); + const kind = requestedKind ?? "html"; + const rawWidgetCode = + readToolStringParam(params, "widget_code", { + trim: false, + }) ?? ""; + const rawReport = asOptionalRecord(params.report); + const isReport = + params.report !== undefined && + params.report !== null && + !(rawWidgetCode.trim() && rawReport && Object.keys(rawReport).length === 0); + const title = readToolStringParam(params, "title", { required: true }); + if (!isReport) { + if (!rawWidgetCode.trim()) { + throw new WidgetHtmlInputError("widget_code required"); + } + assertWidgetHtmlSize(rawWidgetCode, WIDGET_CODE_MAX_CHARS, { + inputName: "widget_code", + unit: "characters", + }); + if (kind === "html") { + // Untrimmed so reported line/column match the widget_code the model sent. + const scriptError = findWidgetScriptSyntaxError(rawWidgetCode); + if (scriptError) { + throw new WidgetHtmlInputError( + `widget_code has a JavaScript syntax error in inline script ${scriptError.scriptIndex} at line ${scriptError.line}, column ${scriptError.column}: ${scriptError.message}. Offending line: ${scriptError.snippet}. Fix the script and call show_widget again.`, + ); + } + } + } + const shouldPin = params.pin === true; + if (pinnedOnly && !shouldPin) { + throw new WidgetHtmlInputError("pin=true is required for this pinned-only widget surface"); + } + const capabilities = normalizeBoardWidgetDeclared( + params.capabilities as { netOrigins?: string[]; tools?: string[] } | undefined, + ); + if (capabilities && !shouldPin) { + throw new WidgetHtmlInputError("capabilities require pin=true"); + } + const pinSessionKey = shouldPin ? options.agentSessionKey?.trim() : undefined; + if (shouldPin && !pinSessionKey) { + throw new WidgetHtmlInputError("pin requires an agent session"); + } + const widgetCode = rawWidgetCode.trim(); + const presentation = asOptionalRecord(params.presentation); + if (pinnedOnly && presentation?.target !== undefined) { + throw new WidgetHtmlInputError( + "presentation.target is unavailable for this pinned-only widget surface", + ); + } + const requestedTarget = + readToolStringParam(presentation ?? {}, "target") ?? "assistant_message"; + if ( + isReport && + (!shouldPin || + rawWidgetCode || + requestedKind || + capabilities || + readToolStringParam(presentation ?? {}, "target")) + ) { + throw new WidgetHtmlInputError( + "Reports require pin=true; omit widget_code, kind, capabilities, and presentation.target. Use HTML for an inline or executable widget", + ); + } + const report = isReport ? parseBoardReport(params.report) : undefined; + const registration = + kind === "html" || isReport + ? undefined + : resolveBoardWidgetContentKind(currentPluginRegistry(), kind); + if (kind !== "html" && !isReport && !registration) { + throw new WidgetHtmlInputError( + `widget kind ${JSON.stringify(kind)} is unavailable; enable the plugin that provides it and retry`, + ); + } + if (registration) { + try { + registration.definition.validateSource(widgetCode); + } catch (error) { + throw new WidgetHtmlInputError(`invalid ${kind} widget source: ${String(error)}`); + } + } + const currentPresenterSupportsKind = + currentChannelPresenter?.target === "current_channel" && + currentChannelPresenter.capabilities.sourceKinds.includes(kind); + const wantsCurrentChannel = + requestedTarget === "assistant_message" && currentPresenterSupportsKind; + const wantsNodePanel = requestedTarget === "node_panel"; + if (!inlineAvailable && !wantsCurrentChannel && !wantsNodePanel && !shouldPin) { + throw new WidgetHtmlInputError( + "inline widget hosting is disabled; set pin=true to place the widget on the session dashboard", + ); + } + if (wantsCurrentChannel && currentChannelPresenter?.target === "current_channel") { + const { maxSourceBytes } = currentChannelPresenter.capabilities; + if (maxSourceBytes !== undefined) { + assertWidgetHtmlSize(rawWidgetCode, maxSourceBytes, { inputName: "widget_code" }); + } + } + const composedWidget = registration + ? registration.definition.composeDocument({ + source: widgetCode, + title, + resourceUrls: Object.fromEntries( + registration.definition.resources.paths.map((resourcePath) => [ + resourcePath, + resourcePath, + ]), + ), + promptGranted: false, + }) + : widgetCode; + const wrappedDocument = isReport + ? "" + : buildWidgetDocument( + title, + composedWidget, + registration ? { scriptOrigins: ["'self'"] } : {}, + ); + let pinnedText = ""; + let pinnedWidgetName: string | undefined; + let capabilityState: BoardWidgetPutResult["widgets"][number]["grantState"] | undefined; + if (pinSessionKey) { + const explicitName = readToolStringParam(params, "name"); + const name = explicitName ?? slugWidgetName(title); + const tab = readToolStringParam(params, "tab"); + const size = readToolStringParam(params, "size"); + const frame = readToolStringParam(presentation ?? {}, "frame"); + const after = readToolStringParam(params, "after"); + const pinnedTitle = boardWidgetTitle(title); + if (!registration && !isReport) { + assertPinnedWidgetDocumentSize( + buildWidgetDocument(pinnedTitle ?? name, widgetCode, { + connectOrigins: capabilities?.netOrigins, + }), + ); + } + const snapshot = await gatewayCall("board.widget.put", { + sessionKey: pinSessionKey, + agentId: options.agentId, + name, + ...(pinnedTitle ? { title: pinnedTitle } : {}), + // The Gateway owns the board document shell so agent-authored bytes + // can never run before its user-activation and bridge bootstrap. + content: report + ? { kind: "plugin", pluginKind: BOARD_REPORT_WIDGET_KIND, props: report } + : registration + ? { kind: "registered", contentKind: kind, source: widgetCode } + : { kind: "html", html: widgetCode }, + ...(frame ? { presentation: frame } : {}), + ...(capabilities ? { declared: capabilities } : {}), + ...(!explicitName ? { generatedIdentity: generatedWidgetIdentity(title, name) } : {}), + ...(tab || size || after + ? { + placement: { + ...(tab ? { tabId: tab } : {}), + ...(size ? { size } : {}), + ...(after ? { after } : {}), + }, + } + : {}), + }); + pinnedWidgetName = snapshot.resolvedWidgetName; + const widget = snapshot.widgets.find( + (candidate) => candidate.name === snapshot.resolvedWidgetName, + ); + if (!widget) { + throw new WidgetHtmlInputError( + "Dashboard did not return the pinned widget; read the board and retry.", + ); + } + capabilityState = widget.grantState; + pinnedText = `pinned to dashboard tab ${widget.tabId} as ${ + snapshot.resolvedWidgetName + }${size ? ` (${size})` : ""}`; + if (capabilityState === "pending") { + pinnedText += + "; capabilities pending: ask the operator to review and approve the dashboard permission card"; + } + if (capabilityState === "rejected") { + pinnedText += + "; capabilities rejected: review the requested access and session permission policy with the operator before retrying"; + } + if (capabilityState === "granted") { + pinnedText += "; capabilities granted"; + } + } + const hasPresentationRoute = + !isReport && (inlineAvailable || wantsCurrentChannel || wantsNodePanel); + if (!hasPresentationRoute) { + return jsonResult({ + status: + capabilityState === "pending" || capabilityState === "rejected" + ? capabilityState + : "pinned", + boardWidgetName: pinnedWidgetName, + capabilityState, + text: `Widget ${pinnedText}. Open this dashboard tab in Control UI to view it.`, + }); + } + let document: Awaited> | undefined; + const hostDocument = async () => + (document ??= await createCanvasDocument( + { + kind: "html_bundle", + title, + entrypoint: { type: "html", value: wrappedDocument }, + surface: "assistant_message", + retentionScope: resolveRetentionScope(options), + // Direct navigation must not run widget script as the Control UI origin. + cspSandbox: "scripts", + }, + { + stateDir: options.stateDir, + maxDocumentsPerScope: WIDGET_MAX_PER_SCOPE, + }, + )); + + let presentationAttempt: WidgetPresentationAttempt | undefined; + if (wantsCurrentChannel && currentChannelPresenter) { + presentationAttempt = await presentWidget({ + presenter: currentChannelPresenter, + document: { kind: "html", html: wrappedDocument }, + title, + context: presenterContext, + }); + } else if (wantsNodePanel) { + const hosted = await hostDocument(); + presentationAttempt = await presentWidget({ + presenter: explicitPresenters.find((presenter) => presenter.target === "node_panel"), + document: { kind: "html", html: wrappedDocument, hostedUrl: hosted.entryUrl }, + title, + context: presenterContext, + }); + } + + if (presentationAttempt?.ok && presentationAttempt.value.kind === "message") { + const receipt = presentationAttempt.value.receipt; + const messageId = receipt.primaryPlatformMessageId ?? receipt.platformMessageIds[0]; + return jsonResult({ + kind: "widget", + presentation: { + target: "current_channel", + title, + receipt, + }, + ...(pinnedWidgetName ? { boardWidgetName: pinnedWidgetName } : {}), + ...(capabilityState ? { capabilityState } : {}), + text: `Widget presented in the current channel${messageId ? ` as message ${messageId}` : ""}${pinnedText ? `; ${pinnedText}` : ""}`, + }); + } + + if (presentationAttempt && !presentationAttempt.ok && !inlineAvailable) { + const failureText = widgetPresentationFailureText(presentationAttempt.error, false); + if (pinnedWidgetName) { + return jsonResult({ + status: "partial", + boardWidgetName: pinnedWidgetName, + capabilityState, + presentation: { + target: requestedTarget === "node_panel" ? "node_panel" : "current_channel", + status: "failed", + error: presentationAttempt.error, + }, + text: `Widget ${pinnedText}, but presentation failed: ${failureText}`, + }); + } + throw new WidgetHtmlInputError(`Widget presentation failed: ${failureText}`); + } + + const hosted = await hostDocument(); + const presentedNode = + presentationAttempt?.ok && presentationAttempt.value.kind === "node" + ? presentationAttempt.value + : undefined; + const target = presentedNode ? "node_panel" : "assistant_message"; + const presentationText = presentedNode + ? `; presented on ${presentedNode.nodeName ?? presentedNode.nodeId} (${presentedNode.nodeId})` + : presentationAttempt && !presentationAttempt.ok + ? `; ${widgetPresentationFailureText(presentationAttempt.error, true)}` + : ""; + return jsonResult({ + kind: "canvas", + ...(capabilityState ? { capabilityState } : {}), + presentation: { + target, + title, + sandbox: "scripts", + ...(presentedNode + ? { node: { id: presentedNode.nodeId, name: presentedNode.nodeName } } + : {}), + }, + view: { + id: hosted.id, + url: hosted.entryUrl, + ...(pinnedWidgetName ? { boardWidgetName: pinnedWidgetName } : {}), + }, + text: `Widget hosted at ${hosted.entryUrl}${pinnedText ? `; ${pinnedText}` : ""}${presentationText}`, + }); + }, + }; +} diff --git a/src/canvas/wrap.test.ts b/src/canvas/wrap.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..eb6caaf37718dc3f1867dccd80538f0b9c518921 --- /dev/null +++ b/src/canvas/wrap.test.ts @@ -0,0 +1,123 @@ +// Widget document wrapper: byte stability and the host-bridge contract it emits. +import { createHash } from "node:crypto"; +import { runInNewContext } from "node:vm"; +import { describe, expect, it, vi } from "vitest"; +import { buildWidgetDocument } from "./wrap.js"; + +describe("buildWidgetDocument", () => { + it("reports bounded runtime errors before widget code, deduplicating and limiting reports", () => { + const html = buildWidgetDocument("Failure", ''); + const bridge = [...html.matchAll(/"; + // This bridge precedes widget code and snapshots every authority-bearing + // primitive. Inline chat keeps its private prompt port; board hosting adopts + // the view ticket and routes every host API over one request channel. + // The activation listeners are that channel's navigation half: the frame + // sandbox grants no popups, so only a trusted new-tab activation on a + // ticketed board widget's http(s) anchor reaches the host. Widening the + // trusted/ticketed/http(s) gates hands ungranted widgets an outbound channel + // that the `connect-src` CSP in board-sandbox.ts otherwise denies. The + // activation predicate mirrors ui/src/app/native-link-routing.ts so widget + // links honor the same primary-click and middle-button contract; listening on + // bubble rather than capture keeps a widget's own preventDefault effective. + const widgetBridge = + ''; + const errorBridge = + "'; + /* + * The host may push a new theme after every theme change. Each message is a + * full snapshot: omitted or invalid tokens are removed so a theme switch + * falls back to the baked palette instead of keeping stale inline values. + * Only the fixed widget token allowlist crosses the frame boundary, and the + * native style setters are captured before widget code can patch them. + */ + const themeBridge = + "'; + const chatHostBridge = + "'; + /* + * Snapshot requests come from the embedding parent and replies return only + * there. Capture the response channel and rendering primitives before widget + * code can patch them; the widget still owns the document being rendered. + */ + const snapshotBridge = + "'; + const connectSources = options.connectOrigins?.length + ? options.connectOrigins.join(" ") + : "'none'"; + const cdnSources = WIDGET_CDN_ORIGINS.join(" "); + const scriptSources = [...WIDGET_CDN_ORIGINS, ...(options.scriptOrigins ?? [])].join(" "); + return ` +${escapeHtml(title)}${widgetBridge}${errorBridge}${themeBridge}${chatHostBridge}${snapshotBridge}${sizeReporter}${widgetCode}`; +} diff --git a/src/cli/account-selector-boundary.test.ts b/src/cli/account-selector-boundary.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..1f80137f616ac39f37c1b2972b5905188ebcd3e7 --- /dev/null +++ b/src/cli/account-selector-boundary.test.ts @@ -0,0 +1,212 @@ +import { Command } from "commander"; +import { describe, expect, it, vi } from "vitest"; +import { registerChannelsCli } from "./channels-cli.js"; +import type { ProgramContext } from "./program/context.js"; +import { registerMessageCommands } from "./program/register.message.js"; + +type AccountOptions = { + account?: string; + channel?: string; + json?: boolean; + dryRun?: boolean; + verbose?: boolean; +}; + +const context: ProgramContext = { + programVersion: "test", + messageChannelOptions: "discord", + agentChannelOptions: "last|discord", +}; + +const messageRoutes: [string, string[]][] = [ + ["send", ["--target", "room", "--message", "hello"]], + ["broadcast", ["--targets", "room", "--message", "hello"]], + ["poll", ["--target", "room", "--poll-question", "Lunch?", "--poll-option", "Soup"]], + ["react", ["--target", "room", "--message-id", "message"]], + ["reactions", ["--target", "room", "--message-id", "message"]], + ["read", ["--target", "room"]], + ["edit", ["--target", "room", "--message-id", "message", "--message", "hello"]], + ["delete", ["--target", "room", "--message-id", "message"]], + ["pin", ["--target", "room", "--message-id", "message"]], + ["unpin", ["--target", "room", "--message-id", "message"]], + ["pins", ["--target", "room"]], + ["permissions", ["--target", "room"]], + ["search", ["--guild-id", "guild", "--query", "hello"]], + ["thread create", ["--target", "room", "--thread-name", "topic"]], + ["thread list", ["--guild-id", "guild"]], + ["thread reply", ["--target", "room", "--message", "hello"]], + ["emoji list", []], + ["emoji upload", ["--guild-id", "guild", "--emoji-name", "smile", "--media", "emoji.png"]], + ["sticker send", ["--target", "room", "--sticker-id", "sticker"]], + [ + "sticker upload", + [ + "--guild-id", + "guild", + "--sticker-name", + "smile", + "--sticker-desc", + "hello", + "--sticker-tags", + "smile", + "--media", + "sticker.png", + ], + ], + ["role info", ["--guild-id", "guild"]], + ["role add", ["--guild-id", "guild", "--user-id", "user", "--role-id", "role"]], + ["role remove", ["--guild-id", "guild", "--user-id", "user", "--role-id", "role"]], + ["channel info", ["--target", "room"]], + ["channel list", ["--guild-id", "guild"]], + ["member info", ["--user-id", "user"]], + ["voice status", ["--guild-id", "guild", "--user-id", "user"]], + ["event list", ["--guild-id", "guild"]], + [ + "event create", + ["--guild-id", "guild", "--event-name", "lunch", "--start-time", "2027-01-01T12:00:00Z"], + ], + ["timeout", ["--guild-id", "guild", "--user-id", "user", "--duration-min", "1"]], + ["kick", ["--guild-id", "guild", "--user-id", "user"]], + ["ban", ["--guild-id", "guild", "--user-id", "user"]], +]; + +const messageCases = messageRoutes.map(([name, required]) => ({ + name: `message ${name}`, + args: [ + "message", + ...name.split(" "), + ...required, + "--channel", + "discord", + "--json", + "--dry-run", + "--verbose", + ], +})); +const channelCases = [ + { + name: "channels capabilities", + args: ["channels", "capabilities", "--channel", "discord", "--json"], + }, + { + name: "channels resolve", + args: ["channels", "resolve", "room", "--channel", "discord", "--json"], + }, +]; +const representativeCases = [ + ...messageCases.filter(({ name }) => + ["message send", "message thread create", "message broadcast"].includes(name), + ), + ...channelCases, +]; + +async function createProgram(args: string[]) { + const program = new Command().exitOverride().configureOutput({ writeErr: () => undefined }); + if (args[0] === "message") { + registerMessageCommands(program, context); + } else { + await registerChannelsCli(program, ["node", "openclaw", ...args]); + } + const startup = vi.fn((_root: Command, _action: Command) => { + throw new Error("command startup reached"); + }); + program.hook("preAction", startup); + return { program, startup }; +} + +describe("account selector option boundaries", () => { + it("covers every registered message action", () => { + const program = new Command(); + registerMessageCommands(program, context); + function leaves(command: Command, prefix: string): string[] { + return command.commands.flatMap((child) => { + const name = `${prefix} ${child.name()}`.trim(); + return child.commands.length ? leaves(child, name) : [name]; + }); + } + expect(leaves(program, "").toSorted()).toEqual(messageCases.map(({ name }) => name).toSorted()); + }); + + describe.each([...messageCases, ...channelCases])("$name", ({ args }) => { + it("preserves valid omitted input before action", async () => { + const { program, startup } = await createProgram(args); + await expect(program.parseAsync(args, { from: "user" })).rejects.toThrow( + "command startup reached", + ); + expect(startup.mock.calls.map(([, action]) => action.opts().account)).toEqual( + [undefined], + ); + }); + + it.each(["", " \t\n "])("rejects blank %j before command startup", async (account) => { + const { program, startup } = await createProgram(args); + await expect( + program.parseAsync([...args, "--account", account], { from: "user" }), + ).rejects.toThrow("--account must not be blank"); + expect(startup).not.toHaveBeenCalled(); + }); + }); + + describe.each(representativeCases)("$name selector forms", ({ args }) => { + it.each([undefined, "work", " work "])( + "preserves nonblank or omitted value %j", + async (account) => { + const { program, startup } = await createProgram(args); + const argv = account === undefined ? args : [...args, "--account", account]; + await expect(program.parseAsync(argv, { from: "user" })).rejects.toThrow( + "command startup reached", + ); + expect( + startup.mock.calls.map(([, action]) => action.opts().account), + ).toEqual([account]); + expect(startup.mock.calls.map(([, action]) => action.opts())).toEqual([ + expect.objectContaining({ + channel: "discord", + json: true, + ...(args[0] === "message" ? { dryRun: true, verbose: true } : {}), + }), + ]); + }, + ); + + it("rejects joined empty input before startup", async () => { + const { program, startup } = await createProgram(args); + await expect(program.parseAsync([...args, "--account="], { from: "user" })).rejects.toThrow( + "--account must not be blank", + ); + expect(startup).not.toHaveBeenCalled(); + }); + + it("keeps the missing option argument error", async () => { + const { program, startup } = await createProgram(args); + await expect(program.parseAsync([...args, "--account"], { from: "user" })).rejects.toThrow( + "argument missing", + ); + expect(startup).not.toHaveBeenCalled(); + }); + }); + + it.each([undefined, "work"])("keeps the missing resolve entry error for %j", async (account) => { + const args = [ + "channels", + "resolve", + "--channel", + "discord", + ...(account === undefined ? [] : ["--account", account]), + ]; + const { program, startup } = await createProgram(args); + await expect(program.parseAsync(args, { from: "user" })).rejects.toThrow( + "missing required argument", + ); + expect(startup).not.toHaveBeenCalled(); + }); + + it("reports a blank selector before a missing resolve entry", async () => { + const args = ["channels", "resolve", "--account", ""]; + const { program, startup } = await createProgram(args); + await expect(program.parseAsync(args, { from: "user" })).rejects.toThrow( + "--account must not be blank", + ); + expect(startup).not.toHaveBeenCalled(); + }); +}); diff --git a/src/cli/acp-cli-exit.process.test.ts b/src/cli/acp-cli-exit.process.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..4e9edd595710b009dcd0d78ff239911f61ecc6d0 --- /dev/null +++ b/src/cli/acp-cli-exit.process.test.ts @@ -0,0 +1,219 @@ +// Process regression coverage for ACP bridge disconnect and startup-handshake exit paths. +import type { ChildProcessWithoutNullStreams } from "node:child_process"; +import { createServer } from "node:http"; +import path from "node:path"; +import { stripVTControlCharacters } from "node:util"; +import { describe, expect, it } from "vitest"; +import { type RawData, WebSocketServer } from "ws"; +import { closeOpenClawStateDatabaseByPath } from "../state/openclaw-state-db-cache.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { createOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { runCliProcessChild } from "./cli-process-child.test-helpers.js"; + +const INITIALIZE_FRAME = { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: 1, + clientCapabilities: { + fs: { readTextFile: false, writeTextFile: false }, + terminal: false, + }, + }, +}; + +async function createPreparedAcpProcessState() { + const state = await createOpenClawTestState({ + applyEnv: false, + label: "acp-process", + scenario: "minimal", + }); + try { + // These cases assert bridge stderr after normal startup. Prepare canonical + // shared state so the one-time migration diagnostic is not part of that signal. + const database = openOpenClawStateDatabase({ env: state.env }); + closeOpenClawStateDatabaseByPath(database.path); + return state; + } catch (error) { + await state.cleanup(); + throw error; + } +} + +function createAcpProcessEnv(baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { + ...baseEnv, + NODE_ENV: undefined, + NODE_OPTIONS: "--use-openssl-ca", + NODE_USE_SYSTEM_CA: "0", + OPENCLAW_DISABLE_BUNDLED_PLUGINS: "1", + OPENCLAW_NO_RESPAWN: "1", + VITEST: undefined, + }; +} + +function withoutSqliteTransactionWarnings(stderr: string): string { + // Slow-hold logger output is load-dependent performance diagnostics, not ACP clean-exit + // signal; see logSlowTransactionHold in sqlite-transaction.ts. + return stderr + .split("\n") + .filter( + (line) => !stripVTControlCharacters(line).trimStart().startsWith("[sqlite/transaction]"), + ) + .join("\n"); +} + +function waitForJsonLine(child: ChildProcessWithoutNullStreams, id: number) { + return new Promise>((resolve, reject) => { + let stdout = ""; + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + reject(new Error(`ACP process exited before response (code=${code}, signal=${signal})`)); + }; + const finish = (response: Record) => { + child.off("exit", onExit); + resolve(response); + }; + + child.once("exit", onExit); + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + const lines = stdout.split("\n"); + stdout = lines.pop() ?? ""; + for (const line of lines) { + if (!line.trim()) { + continue; + } + const response = JSON.parse(line) as Record; + if (response.id === id) { + finish(response); + return; + } + } + }); + }); +} + +function rawDataToText(data: RawData): string { + if (Array.isArray(data)) { + return Buffer.concat(data).toString("utf8"); + } + if (data instanceof ArrayBuffer) { + return Buffer.from(new Uint8Array(data)).toString("utf8"); + } + return Buffer.from(data).toString("utf8"); +} + +describe("ACP CLI process exit", () => { + it("exits when the client disconnects after sending an initialize frame", async () => { + const state = await createPreparedAcpProcessState(); + try { + const result = await runCliProcessChild({ + nodeArgs: [path.resolve("openclaw.mjs"), "acp", "--require-existing"], + env: createAcpProcessEnv(state.env), + input: `${JSON.stringify(INITIALIZE_FRAME)}\n`, + }); + + expect(result).toMatchObject({ code: 0, signal: null }); + expect(withoutSqliteTransactionWarnings(result.stderr)).toBe(""); + } finally { + await state.cleanup(); + } + }); + + it("processes an initialize frame buffered before Gateway hello", async () => { + const state = await createPreparedAcpProcessState(); + const server = createServer(); + const wss = new WebSocketServer({ server }); + + try { + wss.on("connection", (socket) => { + socket.send( + JSON.stringify({ + type: "event", + event: "connect.challenge", + seq: 1, + payload: { nonce: "acp-process-test", ts: Date.now() }, + }), + ); + socket.on("message", (data) => { + const frame = JSON.parse(rawDataToText(data)) as { id: string; method: string }; + if (frame.method !== "connect") { + return; + } + socket.send( + JSON.stringify({ + type: "res", + id: frame.id, + ok: true, + payload: { + type: "hello-ok", + protocol: 4, + server: { version: "acp-process-test", connId: "acp-process-test" }, + features: { methods: [], events: [] }, + snapshot: { + presence: [], + health: {}, + stateVersion: { presence: 1, health: 1 }, + uptimeMs: 1, + }, + auth: { role: "operator", scopes: ["operator.admin"] }, + policy: { + maxPayload: 512 * 1024, + maxBufferedBytes: 1024 * 1024, + tickIntervalMs: 1000, + }, + }, + }), + ); + }); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("ACP process test Gateway did not get a TCP address"); + } + + let response: Record | undefined; + const result = await runCliProcessChild({ + nodeArgs: [ + path.resolve("openclaw.mjs"), + "acp", + "--require-existing", + "--url", + `ws://127.0.0.1:${address.port}`, + ], + env: createAcpProcessEnv(state.env), + interact: async (runningChild) => { + const responsePromise = waitForJsonLine(runningChild, INITIALIZE_FRAME.id); + // Write before the Gateway handshake completes. Startup monitoring must + // retain this frame for the eventual AgentSideConnection reader. + runningChild.stdin.write(`${JSON.stringify(INITIALIZE_FRAME)}\n`); + response = await responsePromise; + runningChild.stdin.end(); + }, + }); + + expect(response).toMatchObject({ + jsonrpc: "2.0", + id: INITIALIZE_FRAME.id, + result: { protocolVersion: INITIALIZE_FRAME.params.protocolVersion }, + }); + expect(result).toMatchObject({ code: 0, signal: null }); + expect(withoutSqliteTransactionWarnings(result.stderr)).toBe(""); + } finally { + for (const socket of wss.clients) { + socket.terminate(); + } + await new Promise((resolve) => { + wss.close(() => resolve()); + }); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + await state.cleanup(); + } + }); +}); diff --git a/src/cli/acp-cli.option-collisions.test.ts b/src/cli/acp-cli.option-collisions.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..aeb3da57e265578cc6660f0dcce90ed44fc243c9 --- /dev/null +++ b/src/cli/acp-cli.option-collisions.test.ts @@ -0,0 +1,202 @@ +// ACP CLI option collision tests cover ACP command flag registration boundaries. +import { Command } from "commander"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { runRegisteredCli } from "../test-utils/command-runner.js"; +import { withTempSecretFiles } from "../test-utils/secret-file-fixture.js"; +import { registerAcpCli } from "./acp-cli.js"; + +type AcpClientOptions = { + verbose?: boolean; +}; + +type AcpGatewayOptions = { + gatewayPassword?: string; + gatewayToken?: string; + prefixCwd?: boolean; +}; + +const mocks = vi.hoisted(() => ({ + runAcpClientInteractive: vi.fn(async (_opts: AcpClientOptions) => {}), + serveAcpGateway: vi.fn(async (_opts: AcpGatewayOptions) => {}), + defaultRuntime: { + log: vi.fn(), + error: vi.fn(), + writeStdout: vi.fn(), + writeJson: vi.fn(), + exit: vi.fn(), + }, +})); + +const { runAcpClientInteractive, serveAcpGateway, defaultRuntime } = mocks; + +const passwordKey = () => ["pass", "word"].join(""); + +vi.mock("../acp/client.js", () => ({ + runAcpClientInteractive: (opts: AcpClientOptions) => mocks.runAcpClientInteractive(opts), +})); + +vi.mock("../acp/server.js", () => ({ + serveAcpGateway: (opts: AcpGatewayOptions) => mocks.serveAcpGateway(opts), +})); + +vi.mock("../runtime.js", () => ({ + defaultRuntime: mocks.defaultRuntime, +})); + +describe("acp cli option collisions", () => { + function createAcpProgram() { + const program = new Command(); + registerAcpCli(program); + return program; + } + + async function parseAcp(args: string[]) { + const program = createAcpProgram(); + await program.parseAsync(["acp", ...args], { from: "user" }); + } + + function expectCliError(pattern: RegExp) { + expect(serveAcpGateway).not.toHaveBeenCalled(); + const errors = defaultRuntime.error.mock.calls.map(([message]) => String(message)); + expect(errors.some((message) => pattern.test(message))).toBe(true); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + } + + function requireFirstMockArg(mock: { mock: { calls: ReadonlyArray> } }) { + const call = mock.mock.calls[0]; + if (!call) { + throw new Error("expected mock to have at least one call"); + } + return call[0]; + } + + beforeEach(() => { + runAcpClientInteractive.mockClear(); + serveAcpGateway.mockClear(); + defaultRuntime.log.mockClear(); + defaultRuntime.error.mockClear(); + defaultRuntime.writeStdout.mockClear(); + defaultRuntime.writeJson.mockClear(); + defaultRuntime.exit.mockClear(); + }); + + it("forwards --verbose to `acp client` when parent and child option names collide", async () => { + await runRegisteredCli({ + register: registerAcpCli as (program: Command) => void, + argv: ["acp", "client", "--verbose"], + }); + + expect(runAcpClientInteractive).toHaveBeenCalledTimes(1); + const clientOptions = requireFirstMockArg(runAcpClientInteractive) as { verbose?: boolean }; + expect(clientOptions?.verbose).toBe(true); + }); + + it("forwards --no-prefix-cwd to the ACP bridge", async () => { + await parseAcp(["--no-prefix-cwd"]); + + expect(serveAcpGateway).toHaveBeenCalledTimes(1); + const gatewayOptions = requireFirstMockArg(serveAcpGateway) as { + prefixCwd?: boolean; + }; + expect(gatewayOptions?.prefixCwd).toBe(false); + }); + + it("defaults to prefixing the working directory", async () => { + await parseAcp([]); + + expect(serveAcpGateway).toHaveBeenCalledTimes(1); + const gatewayOptions = requireFirstMockArg(serveAcpGateway) as { + prefixCwd?: boolean; + }; + expect(gatewayOptions?.prefixCwd).toBe(true); + }); + + it("loads gateway token/password from files", async () => { + await withTempSecretFiles( + "openclaw-acp-cli-", + { token: "tok_file\n", [passwordKey()]: "pw_file\n" }, + async (files) => { + // pragma: allowlist secret + await parseAcp([ + "--token-file", + files.tokenFile ?? "", + "--password-file", + files.passwordFile ?? "", + ]); + }, + ); + + expect(serveAcpGateway).toHaveBeenCalledTimes(1); + const gatewayOptions = requireFirstMockArg(serveAcpGateway) as { + gatewayPassword?: string; + gatewayToken?: string; + }; + expect(gatewayOptions?.gatewayToken).toBe("tok_file"); + expect(gatewayOptions?.gatewayPassword).toBe("pw_file"); // pragma: allowlist secret + }); + + it.each([ + { + name: "rejects mixed secret flags and file flags", + files: { token: "tok_file\n" }, + args: (tokenFile: string) => ["--token", "tok_inline", "--token-file", tokenFile], + expected: /Use either --token .*--token-file for Gateway token\./, + }, + { + name: "rejects mixed password flags and file flags", + files: { password: "pw_file\n" }, // pragma: allowlist secret + args: (_tokenFile: string, passwordFile: string) => [ + "--password", + "pw_inline", + "--password-file", + passwordFile, + ], + expected: /Use either --passw.*d .*--password-file for Gateway password\./, + }, + ])("$name", async ({ files, args, expected }) => { + await withTempSecretFiles("openclaw-acp-cli-", files, async ({ tokenFile, passwordFile }) => { + await parseAcp(args(tokenFile ?? "", passwordFile ?? "")); + }); + + expectCliError(expected); + }); + + it("warns when inline secret flags are used", async () => { + await parseAcp(["--token", "tok_inline", "--password", "pw_inline"]); + + const errors = defaultRuntime.error.mock.calls.map(([message]) => String(message)); + expect(errors).toContain( + "Warning: --token can be exposed via process listings. Prefer --token-file or environment variables.", + ); + expect(errors).toContain( + "Warning: --password can be exposed via process listings. Prefer --password-file or environment variables.", + ); + }); + + it("trims token file path before reading", async () => { + await withTempSecretFiles("openclaw-acp-cli-", { token: "tok_file\n" }, async (files) => { + await parseAcp(["--token-file", ` ${files.tokenFile ?? ""} `]); + }); + + expect(serveAcpGateway).toHaveBeenCalledTimes(1); + const gatewayOptions = requireFirstMockArg(serveAcpGateway) as { gatewayToken?: string }; + expect(gatewayOptions?.gatewayToken).toBe("tok_file"); + }); + + it("reports missing token-file read errors", async () => { + await parseAcp(["--token-file", "/tmp/openclaw-acp-missing-token.txt"]); + expectCliError(/Failed to (inspect|read) Gateway token file/); + }); + + it("formats client errors with formatErrorMessage instead of String(err) (#83904)", async () => { + runAcpClientInteractive.mockImplementationOnce(async () => { + throw { code: 42, why: "boom" } as unknown as Error; + }); + const program = createAcpProgram(); + await program.parseAsync(["acp", "client"], { from: "user" }); + + const errors = defaultRuntime.error.mock.calls.map(([message]) => String(message)); + expect(errors).toContain('{"code":42,"why":"boom"}'); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + }); +}); diff --git a/src/cli/acp-cli.ts b/src/cli/acp-cli.ts new file mode 100644 index 0000000000000000000000000000000000000000..e6e30a32ea4d80aa67a716e8b061cb43f3cdbca1 --- /dev/null +++ b/src/cli/acp-cli.ts @@ -0,0 +1,81 @@ +// Commander registration for ACP bridge and interactive ACP client commands. +import type { Command } from "commander"; +import { formatDocsLink } from "../../packages/terminal-core/src/links.js"; +import { theme } from "../../packages/terminal-core/src/theme.js"; +import { normalizeAcpProvenanceMode } from "../acp/types.js"; +import { formatErrorMessage } from "../infra/errors.js"; +import { defaultRuntime } from "../runtime.js"; +import { inheritOptionFromParent } from "./command-options.js"; +import { resolveGatewayAuthOptions } from "./gateway-secret-options.js"; + +export function registerAcpCli(program: Command) { + const acp = program.command("acp").description("Run an ACP bridge backed by the Gateway"); + + acp + .option("--url ", "Gateway WebSocket URL (defaults to gateway.remote.url when configured)") + .option("--token ", "Gateway token (if required)") + .option("--token-file ", "Read gateway token from file") + .option("--password ", "Gateway password (if required)") + .option("--password-file ", "Read gateway password from file") + .option("--session ", "Default session key (e.g. agent:main:main)") + .option("--session-label