openclaw / src /claws /export.ts
SaylorTwift's picture
SaylorTwift HF Staff
Add files using upload-large-folder tool
d197cf3 verified
Raw History Blame Contribute Delete
24.5 kB
import { createHash } from "node:crypto";
import { closeSync } from "node:fs";
import { mkdir, realpath, rm } from "node:fs/promises";
import { basename, dirname, relative, resolve, sep } from "node:path";
import { coerceErrorMessage } from "@openclaw/normalization-core/error-coercion";
import { stringify as stringifyYaml } from "yaml";
import { listAgentEntries, resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
import { openLocalAgentAvatarFile } from "../agents/identity-avatar-file.js";
import { MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES } from "../agents/workspace-bootstrap-read.js";
import { normalizeConfiguredMcpServers } from "../config/mcp-config-normalize.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { readFileDescriptorBoundedSync } from "../infra/boundary-file-read.js";
import { FsSafeError, root as fsSafeRoot } from "../infra/fs-safe.js";
import { AVATAR_MAX_BYTES, isAvatarDataUrl, isAvatarHttpUrl } from "../shared/avatar-policy.js";
import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js";
import { resolveUserPath } from "../utils.js";
import { readClawStatus } from "./lifecycle-state.js";
import type { PackageRemovalDeps } from "./package-remove.js";
import { readClawManifestFile } from "./reader.js";
import { isPortableClawAvatar } from "./schema-portability.js";
import { parseClawManifest, parseClawOpenClawProfile } from "./schema.js";
import { MAX_CLAW_MANIFEST_BYTES, MAX_MANAGED_WORKSPACE_BYTES } from "./source-limits.js";
import { materializeClawToolProfile } from "./tool-profile-consent.js";
import {
CLAW_BOOTSTRAP_FILE_NAMES,
CLAW_OUTPUT_STABILITY,
CLAW_SCHEMA_VERSION,
type ClawManifest,
type ClawMcpServer,
type ClawOpenClawExtension,
type ClawOpenClawProfile,
type ClawPackagePreflight,
} from "./types.js";
export const CLAW_EXPORT_RESULT_SCHEMA_VERSION = "openclaw.clawExportResult.v1" as const;
const MAX_EXPORT_FILE_BYTES = 1024 * 1024;
type AgentConfig = NonNullable<NonNullable<OpenClawConfig["agents"]>["list"]>[number];
type ClawBootstrapFileName = (typeof CLAW_BOOTSTRAP_FILE_NAMES)[number];
function decodeUtf8(content: Buffer): string | undefined {
try {
return new TextDecoder("utf-8", { fatal: true }).decode(content);
} catch {
return undefined;
}
}
type ClawExportResult = {
schemaVersion: typeof CLAW_EXPORT_RESULT_SCHEMA_VERSION;
stability: typeof CLAW_OUTPUT_STABILITY;
agentId: string;
outputDirectory: string;
manifest: ClawManifest;
openClawProfile?: ClawOpenClawProfile;
filesWritten: string[];
};
const DRIFTED_BOOTSTRAP_STATES = new Set<string>(["modified", "unsafe", "unknown"]);
export class ClawExportError extends Error {
constructor(
readonly code: string,
message: string,
) {
super(message);
this.name = "ClawExportError";
}
}
function portableAgent(agent: AgentConfig, avatar: string | undefined): ClawManifest["agent"] {
const identity = {
...(agent.identity?.name ? { name: agent.identity.name } : {}),
...(agent.identity?.theme ? { theme: agent.identity.theme } : {}),
...(agent.identity?.emoji ? { emoji: agent.identity.emoji } : {}),
...(avatar ? { avatar } : {}),
};
return {
id: agent.id,
...(agent.name ? { name: agent.name } : {}),
...(agent.description ? { description: agent.description } : {}),
...(Object.keys(identity).length > 0 ? { identity } : {}),
};
}
function portableOpenClawProfile(
agent: AgentConfig,
extensions: ClawOpenClawExtension[],
): ClawOpenClawProfile | undefined {
const configuredTools = {
...(agent.tools?.profile ? { profile: agent.tools.profile } : {}),
...(agent.tools?.allow?.length ? { allow: agent.tools.allow } : {}),
...(agent.tools?.alsoAllow?.length ? { alsoAllow: agent.tools.alsoAllow } : {}),
...(agent.tools?.deny?.length ? { deny: agent.tools.deny } : {}),
...(agent.tools?.fs?.workspaceOnly === true ? { fs: { workspaceOnly: true as const } } : {}),
};
let tools: NonNullable<ClawOpenClawProfile["agent"]["tools"]> = configuredTools;
if (configuredTools.profile || configuredTools.allow?.length) {
try {
tools = materializeClawToolProfile({ tools: configuredTools }).tools ?? {};
} catch (error) {
throw new ClawExportError(
"tool_profile_consent_required",
`Could not freeze the exported tool profile: ${(error as Error).message}`,
);
}
}
const settings = {
...(agent.model !== undefined
? { model: typeof agent.model === "string" ? { primary: agent.model } : agent.model }
: {}),
...(agent.subagents
? {
subagents: {
...(agent.subagents.allowAgents !== undefined
? { allowAgents: agent.subagents.allowAgents }
: {}),
...(agent.subagents.delegationMode !== undefined
? { delegationMode: agent.subagents.delegationMode }
: {}),
},
}
: {}),
...(agent.groupChat?.mentionPatterns?.length
? { groupChat: { mentionPatterns: agent.groupChat.mentionPatterns } }
: {}),
...(agent.sandbox
? {
sandbox: {
...(agent.sandbox.mode ? { mode: agent.sandbox.mode } : {}),
...(agent.sandbox.scope ? { scope: agent.sandbox.scope } : {}),
...(agent.sandbox.workspaceAccess
? { workspaceAccess: agent.sandbox.workspaceAccess }
: {}),
},
}
: {}),
...(Object.keys(tools).length > 0 ? { tools } : {}),
...(agent.memory?.search
? {
memory: {
search: {
...(agent.memory.search.enabled !== undefined
? { enabled: agent.memory.search.enabled }
: {}),
...(agent.memory.search.rememberAcrossConversations !== undefined
? {
rememberAcrossConversations: agent.memory.search.rememberAcrossConversations,
}
: {}),
...(agent.memory.search.sources?.length
? { sources: agent.memory.search.sources }
: {}),
},
},
}
: {}),
...(agent.heartbeat
? {
heartbeat: {
...(agent.heartbeat.every ? { every: agent.heartbeat.every } : {}),
...(agent.heartbeat.activeHours
? {
activeHours: {
...(agent.heartbeat.activeHours.start
? { start: agent.heartbeat.activeHours.start }
: {}),
...(agent.heartbeat.activeHours.end
? { end: agent.heartbeat.activeHours.end }
: {}),
...(agent.heartbeat.activeHours.timezone
? { timezone: agent.heartbeat.activeHours.timezone }
: {}),
},
}
: {}),
...(agent.heartbeat.lightContext !== undefined
? { lightContext: agent.heartbeat.lightContext }
: {}),
...(agent.heartbeat.isolatedSession !== undefined
? { isolatedSession: agent.heartbeat.isolatedSession }
: {}),
...(agent.heartbeat.timeoutSeconds !== undefined
? { timeoutSeconds: agent.heartbeat.timeoutSeconds }
: {}),
},
}
: {}),
...(agent.humanDelay
? {
humanDelay: {
...(agent.humanDelay.mode ? { mode: agent.humanDelay.mode } : {}),
...(agent.humanDelay.minMs !== undefined ? { minMs: agent.humanDelay.minMs } : {}),
...(agent.humanDelay.maxMs !== undefined ? { maxMs: agent.humanDelay.maxMs } : {}),
},
}
: {}),
};
if (extensions.length === 0 && Object.keys(settings).length === 0) {
return undefined;
}
const parsed = parseClawOpenClawProfile({ schemaVersion: 1, agent: settings, extensions });
if (!parsed.ok) {
throw new ClawExportError(
"export_openclaw_profile_invalid",
parsed.diagnostics.map((diagnostic) => diagnostic.message).join("; "),
);
}
return parsed.profile;
}
function normalizedRelativePath(value: string): string {
return value.split(sep).join("/");
}
function comparePortableText(left: string, right: string): number {
return left < right ? -1 : left > right ? 1 : 0;
}
function isClawBootstrapFileName(value: string): value is ClawBootstrapFileName {
return (CLAW_BOOTSTRAP_FILE_NAMES as readonly string[]).includes(value);
}
function readPortableAvatar(params: {
config: OpenClawConfig;
agent: AgentConfig;
workspace: string;
}): { source?: string; sidecar?: { path: string; content: Buffer } } {
const source = params.agent.identity?.avatar?.trim();
if (!source) {
return {};
}
if (isAvatarHttpUrl(source)) {
return {};
}
if (isAvatarDataUrl(source)) {
return isPortableClawAvatar(source) ? { source } : {};
}
const opened = openLocalAgentAvatarFile({
cfg: params.config,
agentId: params.agent.id,
source,
});
if (!opened.ok) {
return {};
}
try {
const content = readFileDescriptorBoundedSync(opened.file.fd, AVATAR_MAX_BYTES);
const path = normalizedRelativePath(relative(params.workspace, opened.file.path));
return { source: path, sidecar: { path, content } };
} catch {
return {};
} finally {
closeSync(opened.file.fd);
}
}
function derivativePackageVersion(manifest: ClawManifest, contents: ExportContent[]): string {
const hash = createHash("sha256").update(JSON.stringify(manifest));
for (const file of contents.toSorted((left, right) =>
comparePortableText(left.path, right.path),
)) {
hash.update(file.path).update("\0").update(file.content).update("\0");
}
return `0.0.0-export.${hash.digest("hex")}`;
}
type ExportContent = { path: string; content: Buffer };
async function readAuthorBootstrap(path: string): Promise<Buffer> {
const resolvedPath = resolve(resolveUserPath(path));
try {
const sourceRoot = await fsSafeRoot(dirname(resolvedPath));
const read = await sourceRoot.read(basename(resolvedPath), {
hardlinks: "reject",
maxBytes: MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES,
nonBlockingRead: true,
symlinks: "reject",
});
const text = new TextDecoder("utf-8", { fatal: true }).decode(read.buffer);
if (text.trim().length === 0) {
throw new ClawExportError(
"bootstrap_empty",
"Export BOOTSTRAP.md must contain reviewed first-run instructions.",
);
}
return read.buffer;
} catch (error) {
if (error instanceof ClawExportError) {
throw error;
}
const tooLarge = error instanceof FsSafeError && error.code === "too-large";
throw new ClawExportError(
tooLarge ? "bootstrap_oversized" : "bootstrap_invalid",
tooLarge
? `Export BOOTSTRAP.md exceeds ${MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES} bytes.`
: `Could not read a safe UTF-8 BOOTSTRAP.md from ${JSON.stringify(resolvedPath)}: ${(error as Error).message}`,
);
}
}
function portableMcpServer(server: Record<string, unknown>): ClawMcpServer {
const common = {
...(server.toolFilter && typeof server.toolFilter === "object"
? { toolFilter: server.toolFilter as ClawMcpServer["toolFilter"] }
: {}),
...(typeof server.timeout === "number" ? { timeout: server.timeout } : {}),
...(typeof server.connectTimeout === "number" ? { connectTimeout: server.connectTimeout } : {}),
};
if (typeof server.url === "string") {
if (server.transport !== "sse" && server.transport !== "streamable-http") {
throw new Error("Managed remote MCP server has an unsupported transport.");
}
return {
url: server.url,
transport: server.transport,
...(server.auth === "oauth" ? { auth: "oauth" as const } : {}),
...common,
};
}
if (typeof server.command !== "string") {
throw new Error("Managed MCP server has neither a command nor a remote URL.");
}
return {
command: server.command,
...(server.transport === "stdio" ? { transport: server.transport } : {}),
...(Array.isArray(server.args) ? { args: server.args as string[] } : {}),
...(server.env && typeof server.env === "object"
? { env: server.env as Record<string, string> }
: {}),
...common,
};
}
export async function exportClawAgent(
agentId: string,
outputDirectory: string,
options: OpenClawStateDatabaseOptions & {
config: OpenClawConfig;
packageDeps?: PackageRemovalDeps;
packagePreflight?: ClawPackagePreflight;
sourceMcpServers?: Record<string, Record<string, unknown>>;
bootstrapPath?: string;
},
): Promise<ClawExportResult> {
const status = await readClawStatus(agentId, options);
const record = status.records.find((candidate) => candidate.install.agentId === agentId);
if (!record) {
throw new ClawExportError(
"claw_not_found",
`No installed Claw agent matches ${JSON.stringify(agentId)}.`,
);
}
if (record.install.status !== "complete") {
throw new ClawExportError(
"install_incomplete",
`Installed Claw agent ${JSON.stringify(agentId)} is in ${JSON.stringify(record.install.status)} state; finish or repair it before export.`,
);
}
const agent = listAgentEntries(options.config).find((candidate) => candidate.id === agentId);
if (!agent) {
throw new ClawExportError(
"agent_missing",
`Installed Claw agent ${JSON.stringify(agentId)} is missing from config.`,
);
}
const currentWorkspace = await realpath(
resolve(resolveAgentWorkspaceDir(options.config, agentId)),
).catch(() => resolve(resolveAgentWorkspaceDir(options.config, agentId)));
if (currentWorkspace !== record.install.workspace) {
throw new ClawExportError(
"workspace_changed",
`Agent ${JSON.stringify(agentId)} now resolves to workspace ${JSON.stringify(currentWorkspace)} instead of its recorded Claw workspace ${JSON.stringify(record.install.workspace)}.`,
);
}
if (record.agentState !== "present") {
throw new ClawExportError(
"agent_drifted",
`Agent ${JSON.stringify(agentId)} no longer matches its recorded Claw configuration.`,
);
}
const driftedFiles = record.workspaceFiles.filter((file) => file.state !== "unchanged");
if (driftedFiles.length > 0) {
throw new ClawExportError(
"workspace_files_drifted",
`Cannot export drifted managed files: ${driftedFiles.map((file) => `${file.path} (${file.state})`).join(", ")}.`,
);
}
const driftedPackages = record.packages.filter(
(pkg) =>
pkg.state !== "present" ||
(pkg.extensionCompatibility !== undefined &&
pkg.extensionCompatibility.state !== "compatible"),
);
if (driftedPackages.length > 0) {
throw new ClawExportError(
"packages_drifted",
`Cannot export drifted packages: ${driftedPackages.map((pkg) => `${pkg.kind}:${pkg.ref}@${pkg.version} (${pkg.extensionCompatibility?.state ?? pkg.state})`).join(", ")}.`,
);
}
// A drifted package bootstrap is managed state like any other: exporting it
// silently would publish a package with no BOOTSTRAP.md at all. An explicitly
// reviewed --bootstrap replacement is the supported way through.
if (
record.install.bootstrap &&
!options.bootstrapPath &&
DRIFTED_BOOTSTRAP_STATES.has(record.bootstrapState)
) {
throw new ClawExportError(
"bootstrap_drifted",
`Cannot export the package bootstrap ${JSON.stringify(record.bootstrap.path)} in ${JSON.stringify(record.bootstrapState)} state; restore the seeded file or pass a reviewed --bootstrap replacement.`,
);
}
const unresolvedCronJobs = record.cronJobs.filter(
(cron) => cron.status !== "complete" || !cron.schedulerJobId,
);
const unavailableMcpServers = record.mcpServers.filter((server) => server.state !== "present");
if (unavailableMcpServers.length > 0) {
throw new ClawExportError(
"mcp_servers_unavailable",
`Cannot export MCP servers with unresolved ownership or drift: ${unavailableMcpServers
.map((server) => server.name)
.join(", ")}.`,
);
}
if (unresolvedCronJobs.length > 0) {
throw new ClawExportError(
"cron_jobs_unavailable",
`Cannot export cron declarations with unresolved ownership: ${unresolvedCronJobs
.map((cron) => cron.manifestId)
.join(", ")}.`,
);
}
const authorBootstrap = options.bootstrapPath
? await readAuthorBootstrap(options.bootstrapPath)
: undefined;
const workspace = await fsSafeRoot(record.install.workspace, {
hardlinks: "reject",
maxBytes: MAX_EXPORT_FILE_BYTES,
symlinks: "reject",
});
const allContents: ExportContent[] = await Promise.all(
record.workspaceFiles.map(async (file) => ({
path: normalizedRelativePath(file.path),
content: await workspace.readBytes(file.path, { maxBytes: MAX_EXPORT_FILE_BYTES }),
})),
);
const soul = allContents.find((file) => file.path === "SOUL.md");
const decodedSoul = soul ? decodeUtf8(soul.content) : undefined;
let clawMarkdownBody =
soul && decodedSoul !== undefined && decodedSoul.trim().length > 0 ? soul.content : undefined;
const contents = allContents.filter((file) => file !== soul || !clawMarkdownBody);
const avatar = readPortableAvatar({
config: options.config,
agent,
workspace: record.install.workspace,
});
const managedPaths = new Set(contents.map((file) => file.path));
if (avatar.sidecar && !managedPaths.has(avatar.sidecar.path)) {
contents.push(avatar.sidecar);
}
let pendingPackageBootstrap: Buffer | undefined;
if (!authorBootstrap && record.install.bootstrap && record.bootstrapState === "pending") {
try {
pendingPackageBootstrap = await workspace.readBytes("BOOTSTRAP.md", {
maxBytes: MAX_WORKSPACE_BOOTSTRAP_FILE_BYTES,
});
} catch (error) {
throw new ClawExportError(
"bootstrap_drifted",
`Cannot export the package bootstrap because BOOTSTRAP.md changed after inspection: ${(error as Error).message}`,
);
}
const contentDigest = `sha256:${createHash("sha256")
.update(pendingPackageBootstrap)
.digest("hex")}`;
if (contentDigest !== record.install.bootstrap.contentDigest) {
throw new ClawExportError(
"bootstrap_drifted",
"Cannot export the package bootstrap because BOOTSTRAP.md changed after inspection.",
);
}
}
const exportedBootstrap = authorBootstrap ?? pendingPackageBootstrap;
const bootstrapFiles: ClawManifest["workspace"]["bootstrapFiles"] = {};
const files: ClawManifest["workspace"]["files"] = [];
for (const file of contents) {
const source = `workspace/${file.path}`;
if (isClawBootstrapFileName(file.path)) {
bootstrapFiles[file.path] = { source };
} else {
files.push({ source, path: file.path });
}
}
const configuredMcpServers = normalizeConfiguredMcpServers(
options.sourceMcpServers ?? options.config.mcp?.servers,
);
const extensions = record.packages
.filter((pkg) => pkg.extension)
.map((pkg) => ({
id: pkg.extension!.id,
kind: "plugin" as const,
format: pkg.extension!.format,
source: pkg.source,
ref: pkg.ref,
version: pkg.version,
}))
.toSorted((left, right) => comparePortableText(left.id, right.id));
const openClawProfile = portableOpenClawProfile(agent, extensions);
const openClawProfilePath = "profiles/openclaw.yml";
const openClawProfileRaw = openClawProfile
? Buffer.from(stringifyYaml(openClawProfile))
: undefined;
const portablePackages = record.packages
.filter((pkg) => !pkg.extension)
.map((pkg) => ({
kind: pkg.kind,
source: pkg.source,
ref: pkg.ref,
version: pkg.version,
}))
.toSorted((left, right) => {
const leftIdentity = `${left.kind}:${left.ref}:${left.version}`;
const rightIdentity = `${right.kind}:${right.ref}:${right.version}`;
return comparePortableText(leftIdentity, rightIdentity);
});
const manifest: ClawManifest = {
schemaVersion: CLAW_SCHEMA_VERSION,
agent: portableAgent(agent, avatar.source),
workspace: { bootstrapFiles, files },
packages: portablePackages,
mcpServers: Object.fromEntries(
record.mcpServers.map((ref) => [
ref.name,
portableMcpServer(configuredMcpServers[ref.name]!),
]),
),
cronJobs: record.cronJobs
.map((cron) => cron.job)
.toSorted((left, right) => left.id.localeCompare(right.id)),
};
const serializeClawMarkdown = (body: Buffer | undefined) =>
Buffer.concat([Buffer.from(`---\n${stringifyYaml(manifest)}---\n`), ...(body ? [body] : [])]);
let clawMarkdownRaw = serializeClawMarkdown(clawMarkdownBody);
if (clawMarkdownBody && clawMarkdownRaw.byteLength > MAX_CLAW_MANIFEST_BYTES) {
clawMarkdownBody = undefined;
contents.push(soul!);
bootstrapFiles["SOUL.md"] = { source: "workspace/SOUL.md" };
clawMarkdownRaw = serializeClawMarkdown(undefined);
}
if (clawMarkdownRaw.byteLength > MAX_CLAW_MANIFEST_BYTES) {
throw new ClawExportError(
"claw_manifest_oversized",
`Exported CLAW.md exceeds ${MAX_CLAW_MANIFEST_BYTES} bytes.`,
);
}
const aggregateBytes =
contents.reduce((total, file) => total + file.content.byteLength, 0) +
(clawMarkdownBody?.byteLength ?? 0);
if (aggregateBytes > MAX_MANAGED_WORKSPACE_BYTES) {
throw new ClawExportError(
"workspace_files_oversized",
`Exported workspace content exceeds ${MAX_MANAGED_WORKSPACE_BYTES} aggregate bytes.`,
);
}
const parsed = parseClawManifest(manifest);
if (!parsed.ok) {
throw new ClawExportError(
"export_manifest_invalid",
parsed.diagnostics.map((diagnostic) => diagnostic.message).join("; "),
);
}
const target = resolve(resolveUserPath(outputDirectory));
await mkdir(dirname(target), { recursive: true });
try {
await mkdir(target);
} catch (error) {
throw new ClawExportError(
"output_collision",
`Export directory ${JSON.stringify(target)} must not already exist: ${(error as Error).message}`,
);
}
const filesWritten: string[] = [];
try {
const output = await fsSafeRoot(target, {
hardlinks: "reject",
maxBytes: MAX_EXPORT_FILE_BYTES,
symlinks: "reject",
});
for (const file of contents) {
const path = `workspace/${file.path}`;
await output.write(path, file.content, { mkdir: true, overwrite: false });
filesWritten.push(path);
}
if (openClawProfileRaw) {
await output.write(openClawProfilePath, openClawProfileRaw, {
mkdir: true,
overwrite: false,
});
filesWritten.push(openClawProfilePath);
}
const packageJson = {
name: `openclaw-claw-${record.install.agentId}`,
version: derivativePackageVersion(manifest, [
...contents,
...(clawMarkdownBody ? [{ path: "CLAW.md#body", content: clawMarkdownBody }] : []),
...(openClawProfileRaw ? [{ path: openClawProfilePath, content: openClawProfileRaw }] : []),
...(exportedBootstrap ? [{ path: "BOOTSTRAP.md", content: exportedBootstrap }] : []),
]),
type: "module",
openclaw: { claw: "CLAW.md" },
};
await output.write("package.json", Buffer.from(`${JSON.stringify(packageJson, null, 2)}\n`), {
overwrite: false,
});
filesWritten.push("package.json");
await output.write("CLAW.md", clawMarkdownRaw, { overwrite: false });
filesWritten.push("CLAW.md");
if (exportedBootstrap) {
await output.write("BOOTSTRAP.md", exportedBootstrap, { overwrite: false });
filesWritten.push("BOOTSTRAP.md");
}
const reread = await readClawManifestFile(target);
if (!reread.ok) {
throw new ClawExportError(
"export_package_invalid",
reread.diagnostics.map((diagnostic) => diagnostic.message).join("; "),
);
}
} catch (error) {
await rm(target, { recursive: true, force: true }).catch(() => undefined);
if (error instanceof ClawExportError) {
throw error;
}
throw new ClawExportError("export_write_failed", coerceErrorMessage(error));
}
return {
schemaVersion: CLAW_EXPORT_RESULT_SCHEMA_VERSION,
stability: CLAW_OUTPUT_STABILITY,
agentId,
outputDirectory: target,
manifest,
...(openClawProfile ? { openClawProfile } : {}),
filesWritten,
};
}