// Diagnostic stability bundle helpers collect stable diagnostic data for comparison. import fs from "node:fs"; import path from "node:path"; import process from "node:process"; import { expectDefined } from "@openclaw/normalization-core"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { resolveStateDir } from "../config/paths.js"; import type { DiagnosticMemoryPressureEvent, DiagnosticMemoryUsage, } from "../infra/diagnostic-events.js"; import { collectErrorGraphCandidates, formatErrorMessage, isMissingPathError, } from "../infra/errors.js"; import { registerFatalErrorHook } from "../infra/fatal-error-hooks.js"; import { replaceFileAtomicSync } from "../infra/replace-file.js"; import { getDiagnosticStabilitySnapshot, MAX_DIAGNOSTIC_STABILITY_LIMIT, type DiagnosticStabilitySnapshot, } from "./diagnostic-stability.js"; import { redactSensitiveText } from "./redact.js"; import { formatDiagnosticFilenameTimestamp } from "./timestamps.js"; export const DIAGNOSTIC_STABILITY_BUNDLE_VERSION = 1; const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT = MAX_DIAGNOSTIC_STABILITY_LIMIT; const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION = 20; export const MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES = 5 * 1024 * 1024; const SAFE_REASON_CODE = /^[A-Za-z0-9_.:-]{1,120}$/u; const BUNDLE_PREFIX = "openclaw-stability-"; const BUNDLE_SUFFIX = ".json"; const REDACTED_HOSTNAME = ""; const MAX_SAFE_ERROR_MESSAGE_LENGTH = 500; const MAX_SHUTDOWN_ERRORS = 32; const MAX_SHUTDOWN_ERROR_STACK_LENGTH = 8_000; type DiagnosticHeapSpaceSummary = { spaceName: string; spaceSizeBytes: number; spaceUsedBytes: number; spaceAvailableBytes: number; physicalSpaceSizeBytes: number; }; type DiagnosticHeapStatisticsSummary = { totalHeapSizeBytes: number; totalHeapSizeExecutableBytes: number; totalPhysicalSizeBytes: number; totalAvailableSizeBytes: number; usedHeapSizeBytes: number; heapSizeLimitBytes: number; mallocedMemoryBytes: number; externalMemoryBytes: number; }; type DiagnosticActiveResourceSummary = { total: number; byType: Record; }; type DiagnosticCgroupMemorySummary = { version: "v2"; values: Record; events: Record; }; type DiagnosticSessionFileSummary = { relativePath: string; sizeBytes: number; mtimeMs: number; }; type DiagnosticMemoryPressureBundleEvidence = { level: DiagnosticMemoryPressureEvent["level"]; reason: DiagnosticMemoryPressureEvent["reason"]; memory: DiagnosticMemoryUsage; thresholdBytes?: number; rssGrowthBytes?: number; windowMs?: number; heapStatistics?: DiagnosticHeapStatisticsSummary; heapSpaces?: DiagnosticHeapSpaceSummary[]; cgroup?: DiagnosticCgroupMemorySummary; activeResources?: DiagnosticActiveResourceSummary; topSessionFiles?: DiagnosticSessionFileSummary[]; }; type DiagnosticStabilityBundleEvidence = { memoryPressure?: DiagnosticMemoryPressureBundleEvidence; shutdown?: { step: string; errors: Array & { stack?: string }>; }; }; export type DiagnosticStabilityBundle = { version: typeof DIAGNOSTIC_STABILITY_BUNDLE_VERSION; generatedAt: string; reason: string; process: { pid: number; platform: NodeJS.Platform; arch: string; node: string; uptimeMs: number; }; host: { hostname: string; }; error?: { name?: string; code?: string; message?: string; }; evidence?: DiagnosticStabilityBundleEvidence; snapshot: DiagnosticStabilitySnapshot; }; type WriteDiagnosticStabilityBundleResult = | { status: "written"; path: string; bundle: DiagnosticStabilityBundle } | { status: "skipped"; reason: "empty" } | { status: "failed"; error: unknown }; type WriteDiagnosticStabilityBundleOptions = { reason: string; error?: unknown; includeEmpty?: boolean; limit?: number; now?: Date; env?: NodeJS.ProcessEnv; stateDir?: string; retention?: number; evidence?: DiagnosticStabilityBundleEvidence; shutdownStep?: string; }; type DiagnosticStabilityBundleLocationOptions = { env?: NodeJS.ProcessEnv; stateDir?: string; }; type DiagnosticStabilityBundleFile = { path: string; mtimeMs: number; }; export type ReadDiagnosticStabilityBundleResult = | { status: "found"; path: string; mtimeMs: number; bundle: DiagnosticStabilityBundle } | { status: "missing"; dir: string } | { status: "failed"; path?: string; error: unknown }; type DiagnosticStabilityBundleFailureWriteOutcome = | { status: "written"; message: string; path: string } | { status: "failed"; message: string; error: unknown } | { status: "skipped"; reason: "empty" }; type WriteDiagnosticStabilityBundleForFailureOptions = Omit< WriteDiagnosticStabilityBundleOptions, "error" | "includeEmpty" | "reason" >; let fatalHookUnsubscribe: (() => void) | null = null; function normalizeReason(reason: string): string { return SAFE_REASON_CODE.test(reason) ? reason : "unknown"; } function readErrorCode(error: unknown): string | undefined { if (!error || typeof error !== "object" || !("code" in error)) { return undefined; } const code = (error as { code?: unknown }).code; if (typeof code === "string" && SAFE_REASON_CODE.test(code)) { return code; } if (typeof code === "number" && Number.isFinite(code)) { return String(code); } return undefined; } function readErrorName(error: unknown): string | undefined { if (!error || typeof error !== "object" || !("name" in error)) { return undefined; } const name = (error as { name?: unknown }).name; return typeof name === "string" && SAFE_REASON_CODE.test(name) ? name : undefined; } function readErrorMessage(error: unknown): string | undefined { if (!error || typeof error !== "object" || !("message" in error)) { return undefined; } const message = (error as { message?: unknown }).message; if (typeof message !== "string") { return undefined; } const sanitized = redactSensitiveText(message, { mode: "tools" }).replace(/\s+/gu, " ").trim(); if (!sanitized) { return undefined; } return sanitized.length > MAX_SAFE_ERROR_MESSAGE_LENGTH ? `${truncateUtf16Safe(sanitized, MAX_SAFE_ERROR_MESSAGE_LENGTH)}...` : sanitized; } function readSafeErrorMetadata(error: unknown): DiagnosticStabilityBundle["error"] | undefined { const name = readErrorName(error); const code = readErrorCode(error); const message = readErrorMessage(error); if (!name && !code && !message) { return undefined; } return { ...(name ? { name } : {}), ...(code ? { code } : {}), ...(message ? { message } : {}), }; } function readShutdownError(error: unknown) { const metadata = readSafeErrorMetadata(error) ?? (error === null || typeof error !== "object" ? readSafeErrorMetadata({ message: formatErrorMessage(error) }) : undefined); const stack = error && typeof error === "object" && "stack" in error && typeof error.stack === "string" ? truncateUtf16Safe( redactSensitiveText(error.stack, { mode: "tools" }), MAX_SHUTDOWN_ERROR_STACK_LENGTH, ) : undefined; return { ...metadata, ...(stack ? { stack } : {}) }; } function collectShutdownErrors(error: unknown) { let remaining = MAX_SHUTDOWN_ERRORS - 1; const candidates = collectErrorGraphCandidates(error, (current) => { const nested: unknown[] = [ current.cause, ...(Array.isArray(current.errors) ? current.errors.slice(0, remaining) : []), ] .filter((value) => value !== undefined) .slice(0, remaining); remaining -= nested.length; return nested; }); return (candidates.length ? candidates : [error]).map(readShutdownError); } function resolveDiagnosticStabilityBundleDir( options: DiagnosticStabilityBundleLocationOptions = {}, ): string { return path.join( options.stateDir ?? resolveStateDir(options.env ?? process.env), "logs", "stability", ); } function buildBundlePath(dir: string, now: Date, reason: string): string { return path.join( dir, `${BUNDLE_PREFIX}${formatDiagnosticFilenameTimestamp(now)}-${process.pid}-${normalizeReason(reason)}${BUNDLE_SUFFIX}`, ); } function isBundleFile(name: string): boolean { return name.startsWith(BUNDLE_PREFIX) && name.endsWith(BUNDLE_SUFFIX); } function readObject(value: unknown, label: string): Record { if (!value || typeof value !== "object" || Array.isArray(value)) { throw new Error(`Invalid stability bundle: ${label} must be an object`); } return value as Record; } function readRequiredNumber(value: unknown, label: string): number { if (typeof value !== "number" || !Number.isFinite(value)) { throw new Error(`Invalid stability bundle: ${label} must be a finite number`); } return value; } function readOptionalPositiveInteger(value: unknown, label: string): number | undefined { if (value === undefined) { return undefined; } const parsed = readRequiredNumber(value, label); return parsed >= 0 ? Math.floor(parsed) : undefined; } function readTimestampMs(value: unknown, label: string): number { const timestamp = readRequiredNumber(value, label); if (Number.isNaN(new Date(timestamp).getTime())) { throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`); } return timestamp; } function readOptionalNumber(value: unknown, label: string): number | undefined { if (value === undefined) { return undefined; } return readRequiredNumber(value, label); } function readRequiredString(value: unknown, label: string): string { if (typeof value !== "string") { throw new Error(`Invalid stability bundle: ${label} must be a string`); } return value; } function readTimestampString(value: unknown, label: string): string { const timestamp = readRequiredString(value, label); if (Number.isNaN(new Date(timestamp).getTime())) { throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`); } return timestamp; } function readCodeString(value: unknown, label: string): string { const code = readRequiredString(value, label); if (!SAFE_REASON_CODE.test(code)) { throw new Error(`Invalid stability bundle: ${label} must be a safe diagnostic code`); } return code; } function readOptionalCodeString(value: unknown, label: string): string | undefined { if (value === undefined) { return undefined; } const code = readRequiredString(value, label); return SAFE_REASON_CODE.test(code) ? code : undefined; } function assignOptionalFields( target: T, source: Record, label: string, fields: readonly (keyof T & string)[], read: (value: unknown, label: string) => string | number | undefined, ): void { // The fixed order preserves serialized fields and the first failing validation label. for (const key of fields) { const parsed = read(source[key], `${label}.${key}`); if (parsed !== undefined) { (target as Record)[key] = parsed; } } } function readMemoryUsage(value: unknown, label: string): DiagnosticMemoryUsage { const memory = readObject(value, label); return { rssBytes: readRequiredNumber(memory.rssBytes, `${label}.rssBytes`), heapTotalBytes: readRequiredNumber(memory.heapTotalBytes, `${label}.heapTotalBytes`), heapUsedBytes: readRequiredNumber(memory.heapUsedBytes, `${label}.heapUsedBytes`), externalBytes: readRequiredNumber(memory.externalBytes, `${label}.externalBytes`), arrayBuffersBytes: readRequiredNumber(memory.arrayBuffersBytes, `${label}.arrayBuffersBytes`), }; } function readHeapStatistics(value: unknown): DiagnosticHeapStatisticsSummary | undefined { if (value === undefined) { return undefined; } const source = readObject(value, "evidence.memoryPressure.heapStatistics"); const result = {} as DiagnosticHeapStatisticsSummary; assignOptionalFields( result, source, "evidence.memoryPressure.heapStatistics", [ "totalHeapSizeBytes", "totalHeapSizeExecutableBytes", "totalPhysicalSizeBytes", "totalAvailableSizeBytes", "usedHeapSizeBytes", "heapSizeLimitBytes", "mallocedMemoryBytes", "externalMemoryBytes", ], readOptionalPositiveInteger, ); return Object.keys(result).length > 0 ? result : undefined; } function readHeapSpaces(value: unknown): DiagnosticHeapSpaceSummary[] | undefined { if (value === undefined) { return undefined; } if (!Array.isArray(value)) { throw new Error( "Invalid stability bundle: evidence.memoryPressure.heapSpaces must be an array", ); } const spaces: DiagnosticHeapSpaceSummary[] = []; for (const [index, entry] of value.entries()) { const source = readObject(entry, `evidence.memoryPressure.heapSpaces[${index}]`); const spaceName = readOptionalCodeString( source.spaceName, `evidence.memoryPressure.heapSpaces[${index}].spaceName`, ); if (!spaceName) { continue; } spaces.push({ spaceName, spaceSizeBytes: readOptionalPositiveInteger( source.spaceSizeBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceSizeBytes`, ) ?? 0, spaceUsedBytes: readOptionalPositiveInteger( source.spaceUsedBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceUsedBytes`, ) ?? 0, spaceAvailableBytes: readOptionalPositiveInteger( source.spaceAvailableBytes, `evidence.memoryPressure.heapSpaces[${index}].spaceAvailableBytes`, ) ?? 0, physicalSpaceSizeBytes: readOptionalPositiveInteger( source.physicalSpaceSizeBytes, `evidence.memoryPressure.heapSpaces[${index}].physicalSpaceSizeBytes`, ) ?? 0, }); } return spaces.length > 0 ? spaces : undefined; } function readCgroupMemorySummary(value: unknown): DiagnosticCgroupMemorySummary | undefined { if (value === undefined) { return undefined; } const source = readObject(value, "evidence.memoryPressure.cgroup"); const version = readCodeString( source.version, "evidence.memoryPressure.cgroup.version", ) as DiagnosticCgroupMemorySummary["version"]; if (version !== "v2") { return undefined; } const valuesSource = readObject(source.values, "evidence.memoryPressure.cgroup.values"); const values: Record = {}; for (const [key, raw] of Object.entries(valuesSource)) { if (!SAFE_REASON_CODE.test(key)) { continue; } if (raw === "max") { values[key] = "max"; } else { values[key] = readOptionalPositiveInteger(raw, `evidence.memoryPressure.cgroup.values.${key}`) ?? 0; } } return { version, values, events: readNumberMap(source.events, "evidence.memoryPressure.cgroup.events"), }; } function readActiveResources(value: unknown): DiagnosticActiveResourceSummary | undefined { if (value === undefined) { return undefined; } const source = readObject(value, "evidence.memoryPressure.activeResources"); return { total: readOptionalPositiveInteger(source.total, "evidence.memoryPressure.activeResources.total") ?? 0, byType: readNumberMap(source.byType, "evidence.memoryPressure.activeResources.byType"), }; } function readSessionFiles(value: unknown): DiagnosticSessionFileSummary[] | undefined { if (value === undefined) { return undefined; } if (!Array.isArray(value)) { throw new Error( "Invalid stability bundle: evidence.memoryPressure.topSessionFiles must be an array", ); } const files: DiagnosticSessionFileSummary[] = []; for (const [index, entry] of value.entries()) { const source = readObject(entry, `evidence.memoryPressure.topSessionFiles[${index}]`); const relativePath = readRequiredString( source.relativePath, `evidence.memoryPressure.topSessionFiles[${index}].relativePath`, ); if ( path.isAbsolute(relativePath) || relativePath.includes("..") || relativePath.length > 300 || /[\r\n]/u.test(relativePath) ) { continue; } files.push({ relativePath: sanitizeSessionEvidencePath(relativePath), sizeBytes: readOptionalPositiveInteger( source.sizeBytes, `evidence.memoryPressure.topSessionFiles[${index}].sizeBytes`, ) ?? 0, mtimeMs: readOptionalPositiveInteger( source.mtimeMs, `evidence.memoryPressure.topSessionFiles[${index}].mtimeMs`, ) ?? 0, }); } return files.length > 0 ? files : undefined; } function readMemoryPressureEvidence( value: unknown, ): DiagnosticMemoryPressureBundleEvidence | undefined { if (value === undefined) { return undefined; } const pressure = readObject(value, "evidence.memoryPressure"); const level = readCodeString( pressure.level, "evidence.memoryPressure.level", ) as DiagnosticMemoryPressureEvent["level"]; const reason = readCodeString( pressure.reason, "evidence.memoryPressure.reason", ) as DiagnosticMemoryPressureEvent["reason"]; if ((level !== "warning" && level !== "critical") || !isMemoryPressureReason(reason)) { return undefined; } const heapStatistics = readHeapStatistics(pressure.heapStatistics); const heapSpaces = readHeapSpaces(pressure.heapSpaces); const cgroup = readCgroupMemorySummary(pressure.cgroup); const activeResources = readActiveResources(pressure.activeResources); const topSessionFiles = readSessionFiles(pressure.topSessionFiles); const result: DiagnosticMemoryPressureBundleEvidence = { level, reason, memory: readMemoryUsage(pressure.memory, "evidence.memoryPressure.memory"), }; assignOptionalFields( result, pressure, "evidence.memoryPressure", ["thresholdBytes", "rssGrowthBytes", "windowMs"], readOptionalNumber, ); return { ...result, ...(heapStatistics ? { heapStatistics } : {}), ...(heapSpaces ? { heapSpaces } : {}), ...(cgroup ? { cgroup } : {}), ...(activeResources ? { activeResources } : {}), ...(topSessionFiles ? { topSessionFiles } : {}), }; } function readBundleEvidence(value: unknown): DiagnosticStabilityBundleEvidence | undefined { if (value === undefined) { return undefined; } const source = readObject(value, "evidence"); const memoryPressure = readMemoryPressureEvidence(source.memoryPressure); let shutdown: DiagnosticStabilityBundleEvidence["shutdown"]; if (source.shutdown !== undefined) { const shutdownSource = readObject(source.shutdown, "evidence.shutdown"); if (!Array.isArray(shutdownSource.errors)) { throw new Error("Invalid stability bundle: evidence.shutdown.errors must be an array"); } shutdown = { step: readCodeString(shutdownSource.step, "evidence.shutdown.step"), errors: shutdownSource.errors.slice(0, MAX_SHUTDOWN_ERRORS).map(readShutdownError), }; } return memoryPressure || shutdown ? { ...(memoryPressure ? { memoryPressure } : {}), ...(shutdown ? { shutdown } : {}) } : undefined; } function readNumberMap(value: unknown, label: string): Record { const source = readObject(value, label); const result: Record = {}; for (const [key, entry] of Object.entries(source)) { if (!SAFE_REASON_CODE.test(key)) { continue; } result[key] = readRequiredNumber(entry, `${label}.${key}`); } return result; } function readOptionalMemorySummary( value: unknown, ): DiagnosticStabilitySnapshot["summary"]["memory"] | undefined { if (value === undefined) { return undefined; } const memory = readObject(value, "snapshot.summary.memory"); const latest = memory.latest === undefined ? undefined : readMemoryUsage(memory.latest, "snapshot.summary.memory.latest"); return { ...(latest ? { latest } : {}), ...(memory.maxRssBytes !== undefined ? { maxRssBytes: readRequiredNumber( memory.maxRssBytes, "snapshot.summary.memory.maxRssBytes", ), } : {}), ...(memory.maxHeapUsedBytes !== undefined ? { maxHeapUsedBytes: readRequiredNumber( memory.maxHeapUsedBytes, "snapshot.summary.memory.maxHeapUsedBytes", ), } : {}), pressureCount: readRequiredNumber( memory.pressureCount, "snapshot.summary.memory.pressureCount", ), }; } function readOptionalPayloadLargeSummary( value: unknown, ): DiagnosticStabilitySnapshot["summary"]["payloadLarge"] | undefined { if (value === undefined) { return undefined; } const payloadLarge = readObject(value, "snapshot.summary.payloadLarge"); return { count: readRequiredNumber(payloadLarge.count, "snapshot.summary.payloadLarge.count"), rejected: readRequiredNumber(payloadLarge.rejected, "snapshot.summary.payloadLarge.rejected"), truncated: readRequiredNumber( payloadLarge.truncated, "snapshot.summary.payloadLarge.truncated", ), chunked: readRequiredNumber(payloadLarge.chunked, "snapshot.summary.payloadLarge.chunked"), bySurface: readNumberMap(payloadLarge.bySurface, "snapshot.summary.payloadLarge.bySurface"), }; } function readStabilityEventRecord( value: unknown, label: string, ): DiagnosticStabilitySnapshot["events"][number] { const record = readObject(value, label); const sanitized: DiagnosticStabilitySnapshot["events"][number] = { seq: readRequiredNumber(record.seq, `${label}.seq`), ts: readTimestampMs(record.ts, `${label}.ts`), type: readCodeString( record.type, `${label}.type`, ) as DiagnosticStabilitySnapshot["events"][number]["type"], }; assignOptionalFields( sanitized, record, label, [ "channel", "pluginId", "source", "surface", "action", "reason", "outcome", "level", "phase", "approvalId", "detector", "toolName", "activeWorkKind", "pairedToolName", "provider", "model", ], readOptionalCodeString, ); assignOptionalFields( sanitized, record, label, [ "durationMs", "requestBytes", "responseBytes", "timeToFirstByteMs", "costUsd", "count", "bytes", "limitBytes", "thresholdBytes", "rssGrowthBytes", "windowMs", "ageMs", "queueDepth", "queueSize", "queueLength", "waitMs", "active", "waiting", "queued", "droppedEvents", "droppedTrustedEvents", "droppedUntrustedEvents", "droppedPriorityEvents", "maxQueueLength", "drainBatchSize", ], readOptionalNumber, ); if (record.webhooks !== undefined) { const webhooks = readObject(record.webhooks, `${label}.webhooks`); sanitized.webhooks = { received: readRequiredNumber(webhooks.received, `${label}.webhooks.received`), processed: readRequiredNumber(webhooks.processed, `${label}.webhooks.processed`), errors: readRequiredNumber(webhooks.errors, `${label}.webhooks.errors`), }; } if (record.memory !== undefined) { sanitized.memory = readMemoryUsage(record.memory, `${label}.memory`); } if (record.usage !== undefined) { const usage = readObject(record.usage, `${label}.usage`); sanitized.usage = {}; assignOptionalFields( sanitized.usage, usage, `${label}.usage`, ["input", "output", "cacheRead", "cacheWrite", "promptTokens", "total"], readOptionalNumber, ); } if (record.context !== undefined) { const context = readObject(record.context, `${label}.context`); sanitized.context = {}; assignOptionalFields( sanitized.context, context, `${label}.context`, ["limit", "used"], readOptionalNumber, ); } return sanitized; } function readStabilitySnapshot(value: unknown): DiagnosticStabilitySnapshot { const snapshot = readObject(value, "snapshot"); const generatedAt = readTimestampString(snapshot.generatedAt, "snapshot.generatedAt"); const capacity = readRequiredNumber(snapshot.capacity, "snapshot.capacity"); const count = readRequiredNumber(snapshot.count, "snapshot.count"); const dropped = readRequiredNumber(snapshot.dropped, "snapshot.dropped"); const firstSeq = readOptionalNumber(snapshot.firstSeq, "snapshot.firstSeq"); const lastSeq = readOptionalNumber(snapshot.lastSeq, "snapshot.lastSeq"); if (!Array.isArray(snapshot.events)) { throw new Error("Invalid stability bundle: snapshot.events must be an array"); } const events = snapshot.events.map((event, index) => readStabilityEventRecord(event, `snapshot.events[${index}]`), ); const summary = readObject(snapshot.summary, "snapshot.summary"); return { generatedAt, capacity, count, dropped, ...(firstSeq !== undefined ? { firstSeq } : {}), ...(lastSeq !== undefined ? { lastSeq } : {}), events, summary: { byType: readNumberMap(summary.byType, "snapshot.summary.byType"), ...(summary.memory !== undefined ? { memory: readOptionalMemorySummary(summary.memory) } : {}), ...(summary.payloadLarge !== undefined ? { payloadLarge: readOptionalPayloadLargeSummary(summary.payloadLarge) } : {}), }, }; } function parseDiagnosticStabilityBundle(value: unknown): DiagnosticStabilityBundle { const bundle = readObject(value, "bundle"); if (bundle.version !== DIAGNOSTIC_STABILITY_BUNDLE_VERSION) { throw new Error(`Unsupported stability bundle version: ${String(bundle.version)}`); } const processInfo = readObject(bundle.process, "process"); readObject(bundle.host, "host"); const error = bundle.error === undefined ? undefined : readSafeErrorMetadata(bundle.error); const evidence = readBundleEvidence(bundle.evidence); return { version: DIAGNOSTIC_STABILITY_BUNDLE_VERSION, generatedAt: readTimestampString(bundle.generatedAt, "generatedAt"), reason: normalizeReason(readRequiredString(bundle.reason, "reason")), process: { pid: readRequiredNumber(processInfo.pid, "process.pid"), platform: readCodeString(processInfo.platform, "process.platform") as NodeJS.Platform, arch: readCodeString(processInfo.arch, "process.arch"), node: readCodeString(processInfo.node, "process.node"), uptimeMs: readRequiredNumber(processInfo.uptimeMs, "process.uptimeMs"), }, host: { hostname: REDACTED_HOSTNAME, }, ...(error ? { error } : {}), ...(evidence ? { evidence } : {}), snapshot: readStabilitySnapshot(bundle.snapshot), }; } function sanitizeSessionEvidencePath(relativePath: string): string { const parts = relativePath.split("/"); if (parts.length === 4 && parts[0] === "agents" && parts[2] === "sessions") { return `agents//sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[3], "parts entry at 3"))}`; } if (parts.length === 2 && parts[0] === "sessions") { return `sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[1], "parts entry at 1"))}`; } return redactSensitiveText(relativePath, { mode: "tools" }); } function sanitizeSessionEvidenceFileName(fileName: string): string { if (fileName === "sessions.json") { return "sessions.json"; } if (fileName.endsWith(".jsonl")) { return ".jsonl"; } if (fileName.endsWith(".json")) { return ".json"; } return ""; } function isMemoryPressureReason(reason: string): reason is DiagnosticMemoryPressureEvent["reason"] { return reason === "rss_threshold" || reason === "heap_threshold" || reason === "rss_growth"; } function listDiagnosticStabilityBundleFilesSync( options: DiagnosticStabilityBundleLocationOptions = {}, ): DiagnosticStabilityBundleFile[] { const dir = resolveDiagnosticStabilityBundleDir(options); try { return fs .readdirSync(dir, { withFileTypes: true }) .filter((entry) => entry.isFile() && isBundleFile(entry.name)) .map((entry) => { const file = path.join(dir, entry.name); return { path: file, mtimeMs: fs.statSync(file).mtimeMs, }; }) .toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.path.localeCompare(a.path)); } catch (error) { if (isMissingPathError(error)) { return []; } throw error; } } export function readDiagnosticStabilityBundleFileSync( file: string, ): ReadDiagnosticStabilityBundleResult { try { const stat = fs.statSync(file); if (stat.size > MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES) { throw new Error( `Stability bundle is too large: ${stat.size} bytes exceeds ${MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES}`, ); } const raw = fs.readFileSync(file, "utf8"); const bundle = parseDiagnosticStabilityBundle(JSON.parse(raw)); return { status: "found", path: file, mtimeMs: stat.mtimeMs, bundle, }; } catch (error) { return { status: "failed", path: file, error }; } } export function readLatestDiagnosticStabilityBundleSync( options: DiagnosticStabilityBundleLocationOptions = {}, ): ReadDiagnosticStabilityBundleResult { try { const latest = listDiagnosticStabilityBundleFilesSync(options)[0]; if (!latest) { return { status: "missing", dir: resolveDiagnosticStabilityBundleDir(options), }; } return readDiagnosticStabilityBundleFileSync(latest.path); } catch (error) { return { status: "failed", error }; } } function pruneOldBundles(dir: string, retention: number, retainedFile: string): void { if (!Number.isFinite(retention) || retention < 1) { return; } try { const entries = fs .readdirSync(dir, { withFileTypes: true }) .filter((entry) => entry.isFile() && isBundleFile(entry.name)) .map((entry) => { const file = path.join(dir, entry.name); let mtimeMs = 0; try { mtimeMs = fs.statSync(file).mtimeMs; } catch { // Missing files are ignored below. } return { file, mtimeMs }; }) .filter((entry) => entry.file !== retainedFile) .toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.file.localeCompare(a.file)); for (const entry of entries.slice(retention - 1)) { try { fs.unlinkSync(entry.file); } catch { // Retention cleanup must not block failure handling. } } } catch { // Retention cleanup must not block failure handling. } } export function writeDiagnosticStabilityBundleSync( options: WriteDiagnosticStabilityBundleOptions, ): WriteDiagnosticStabilityBundleResult { try { const now = options.now ?? new Date(); const snapshot = getDiagnosticStabilitySnapshot({ limit: options.limit ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT, }); if (!options.includeEmpty && snapshot.count === 0) { return { status: "skipped", reason: "empty" }; } const reason = normalizeReason(options.reason); const error = options.error ? readSafeErrorMetadata(options.error) : undefined; const evidence = options.shutdownStep ? { ...options.evidence, shutdown: { step: readCodeString(options.shutdownStep, "shutdownStep"), errors: collectShutdownErrors(options.error), }, } : options.evidence; const bundle: DiagnosticStabilityBundle = { version: DIAGNOSTIC_STABILITY_BUNDLE_VERSION, generatedAt: now.toISOString(), reason, process: { pid: process.pid, platform: process.platform, arch: process.arch, node: process.versions.node, uptimeMs: Math.round(process.uptime() * 1000), }, host: { hostname: REDACTED_HOSTNAME, }, ...(error ? { error } : {}), ...(evidence ? { evidence } : {}), snapshot, }; const dir = resolveDiagnosticStabilityBundleDir(options); const file = buildBundlePath(dir, now, reason); replaceFileAtomicSync({ filePath: file, content: `${JSON.stringify(bundle, null, 2)}\n`, dirMode: 0o700, mode: 0o600, tempPrefix: ".openclaw-stability", }); pruneOldBundles(dir, options.retention ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION, file); return { status: "written", path: file, bundle }; } catch (error) { return { status: "failed", error }; } } export function writeDiagnosticStabilityBundleForFailureSync( reason: string, error?: unknown, options: WriteDiagnosticStabilityBundleForFailureOptions = {}, ): DiagnosticStabilityBundleFailureWriteOutcome { const result = writeDiagnosticStabilityBundleSync({ ...options, reason, error, includeEmpty: true, }); if (result.status === "written") { return { status: "written", path: result.path, message: `wrote stability bundle: ${result.path}`, }; } if (result.status === "failed") { return { status: "failed", error: result.error, message: `failed to write stability bundle: ${String(result.error)}`, }; } return result; } export function installDiagnosticStabilityFatalHook( options: WriteDiagnosticStabilityBundleForFailureOptions = {}, ): void { if (fatalHookUnsubscribe) { return; } fatalHookUnsubscribe = registerFatalErrorHook(({ reason, error }) => { const result = writeDiagnosticStabilityBundleForFailureSync(reason, error, options); return "message" in result ? result.message : undefined; }); } export function uninstallDiagnosticStabilityFatalHook(): void { fatalHookUnsubscribe?.(); fatalHookUnsubscribe = null; } /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */