openclaw / src /logging /diagnostic-stability-bundle.ts
Add files using upload-large-folder tool
76289e7 verified about 2 months ago
34 kB
// Diagnostic stability bundle helpers collect stable diagnostic data for comparison.
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import { expectDefined } from "@openclaw/normalization-core";
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { resolveStateDir } from "../config/paths.js";
import type {
  DiagnosticMemoryPressureEvent,
  DiagnosticMemoryUsage,
} from "../infra/diagnostic-events.js";
import {
  collectErrorGraphCandidates,
  formatErrorMessage,
  isMissingPathError,
} from "../infra/errors.js";
import { registerFatalErrorHook } from "../infra/fatal-error-hooks.js";
import { replaceFileAtomicSync } from "../infra/replace-file.js";
import {
  getDiagnosticStabilitySnapshot,
  MAX_DIAGNOSTIC_STABILITY_LIMIT,
  type DiagnosticStabilitySnapshot,
} from "./diagnostic-stability.js";
import { redactSensitiveText } from "./redact.js";
import { formatDiagnosticFilenameTimestamp } from "./timestamps.js";

export const DIAGNOSTIC_STABILITY_BUNDLE_VERSION = 1;
const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT = MAX_DIAGNOSTIC_STABILITY_LIMIT;
const DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION = 20;
export const MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES = 5 * 1024 * 1024;

const SAFE_REASON_CODE = /^[A-Za-z0-9_.:-]{1,120}$/u;
const BUNDLE_PREFIX = "openclaw-stability-";
const BUNDLE_SUFFIX = ".json";
const REDACTED_HOSTNAME = "<redacted-hostname>";
const MAX_SAFE_ERROR_MESSAGE_LENGTH = 500;
const MAX_SHUTDOWN_ERRORS = 32;
const MAX_SHUTDOWN_ERROR_STACK_LENGTH = 8_000;

type DiagnosticHeapSpaceSummary = {
  spaceName: string;
  spaceSizeBytes: number;
  spaceUsedBytes: number;
  spaceAvailableBytes: number;
  physicalSpaceSizeBytes: number;
};

type DiagnosticHeapStatisticsSummary = {
  totalHeapSizeBytes: number;
  totalHeapSizeExecutableBytes: number;
  totalPhysicalSizeBytes: number;
  totalAvailableSizeBytes: number;
  usedHeapSizeBytes: number;
  heapSizeLimitBytes: number;
  mallocedMemoryBytes: number;
  externalMemoryBytes: number;
};

type DiagnosticActiveResourceSummary = {
  total: number;
  byType: Record<string, number>;
};

type DiagnosticCgroupMemorySummary = {
  version: "v2";
  values: Record<string, number | "max">;
  events: Record<string, number>;
};

type DiagnosticSessionFileSummary = {
  relativePath: string;
  sizeBytes: number;
  mtimeMs: number;
};

type DiagnosticMemoryPressureBundleEvidence = {
  level: DiagnosticMemoryPressureEvent["level"];
  reason: DiagnosticMemoryPressureEvent["reason"];
  memory: DiagnosticMemoryUsage;
  thresholdBytes?: number;
  rssGrowthBytes?: number;
  windowMs?: number;
  heapStatistics?: DiagnosticHeapStatisticsSummary;
  heapSpaces?: DiagnosticHeapSpaceSummary[];
  cgroup?: DiagnosticCgroupMemorySummary;
  activeResources?: DiagnosticActiveResourceSummary;
  topSessionFiles?: DiagnosticSessionFileSummary[];
};

type DiagnosticStabilityBundleEvidence = {
  memoryPressure?: DiagnosticMemoryPressureBundleEvidence;
  shutdown?: {
    step: string;
    errors: Array<NonNullable<DiagnosticStabilityBundle["error"]> & { stack?: string }>;
  };
};

export type DiagnosticStabilityBundle = {
  version: typeof DIAGNOSTIC_STABILITY_BUNDLE_VERSION;
  generatedAt: string;
  reason: string;
  process: {
    pid: number;
    platform: NodeJS.Platform;
    arch: string;
    node: string;
    uptimeMs: number;
  };
  host: {
    hostname: string;
  };
  error?: {
    name?: string;
    code?: string;
    message?: string;
  };
  evidence?: DiagnosticStabilityBundleEvidence;
  snapshot: DiagnosticStabilitySnapshot;
};

type WriteDiagnosticStabilityBundleResult =
  | { status: "written"; path: string; bundle: DiagnosticStabilityBundle }
  | { status: "skipped"; reason: "empty" }
  | { status: "failed"; error: unknown };

type WriteDiagnosticStabilityBundleOptions = {
  reason: string;
  error?: unknown;
  includeEmpty?: boolean;
  limit?: number;
  now?: Date;
  env?: NodeJS.ProcessEnv;
  stateDir?: string;
  retention?: number;
  evidence?: DiagnosticStabilityBundleEvidence;
  shutdownStep?: string;
};

type DiagnosticStabilityBundleLocationOptions = {
  env?: NodeJS.ProcessEnv;
  stateDir?: string;
};

type DiagnosticStabilityBundleFile = {
  path: string;
  mtimeMs: number;
};

export type ReadDiagnosticStabilityBundleResult =
  | { status: "found"; path: string; mtimeMs: number; bundle: DiagnosticStabilityBundle }
  | { status: "missing"; dir: string }
  | { status: "failed"; path?: string; error: unknown };

type DiagnosticStabilityBundleFailureWriteOutcome =
  | { status: "written"; message: string; path: string }
  | { status: "failed"; message: string; error: unknown }
  | { status: "skipped"; reason: "empty" };

type WriteDiagnosticStabilityBundleForFailureOptions = Omit<
  WriteDiagnosticStabilityBundleOptions,
  "error" | "includeEmpty" | "reason"
>;

let fatalHookUnsubscribe: (() => void) | null = null;

function normalizeReason(reason: string): string {
  return SAFE_REASON_CODE.test(reason) ? reason : "unknown";
}

function readErrorCode(error: unknown): string | undefined {
  if (!error || typeof error !== "object" || !("code" in error)) {
    return undefined;
  }
  const code = (error as { code?: unknown }).code;
  if (typeof code === "string" && SAFE_REASON_CODE.test(code)) {
    return code;
  }
  if (typeof code === "number" && Number.isFinite(code)) {
    return String(code);
  }
  return undefined;
}

function readErrorName(error: unknown): string | undefined {
  if (!error || typeof error !== "object" || !("name" in error)) {
    return undefined;
  }
  const name = (error as { name?: unknown }).name;
  return typeof name === "string" && SAFE_REASON_CODE.test(name) ? name : undefined;
}

function readErrorMessage(error: unknown): string | undefined {
  if (!error || typeof error !== "object" || !("message" in error)) {
    return undefined;
  }
  const message = (error as { message?: unknown }).message;
  if (typeof message !== "string") {
    return undefined;
  }
  const sanitized = redactSensitiveText(message, { mode: "tools" }).replace(/\s+/gu, " ").trim();
  if (!sanitized) {
    return undefined;
  }
  return sanitized.length > MAX_SAFE_ERROR_MESSAGE_LENGTH
    ? `${truncateUtf16Safe(sanitized, MAX_SAFE_ERROR_MESSAGE_LENGTH)}...`
    : sanitized;
}

function readSafeErrorMetadata(error: unknown): DiagnosticStabilityBundle["error"] | undefined {
  const name = readErrorName(error);
  const code = readErrorCode(error);
  const message = readErrorMessage(error);
  if (!name && !code && !message) {
    return undefined;
  }
  return {
    ...(name ? { name } : {}),
    ...(code ? { code } : {}),
    ...(message ? { message } : {}),
  };
}

function readShutdownError(error: unknown) {
  const metadata =
    readSafeErrorMetadata(error) ??
    (error === null || typeof error !== "object"
      ? readSafeErrorMetadata({ message: formatErrorMessage(error) })
      : undefined);
  const stack =
    error && typeof error === "object" && "stack" in error && typeof error.stack === "string"
      ? truncateUtf16Safe(
          redactSensitiveText(error.stack, { mode: "tools" }),
          MAX_SHUTDOWN_ERROR_STACK_LENGTH,
        )
      : undefined;
  return { ...metadata, ...(stack ? { stack } : {}) };
}

function collectShutdownErrors(error: unknown) {
  let remaining = MAX_SHUTDOWN_ERRORS - 1;
  const candidates = collectErrorGraphCandidates(error, (current) => {
    const nested: unknown[] = [
      current.cause,
      ...(Array.isArray(current.errors) ? current.errors.slice(0, remaining) : []),
    ]
      .filter((value) => value !== undefined)
      .slice(0, remaining);
    remaining -= nested.length;
    return nested;
  });
  return (candidates.length ? candidates : [error]).map(readShutdownError);
}

function resolveDiagnosticStabilityBundleDir(
  options: DiagnosticStabilityBundleLocationOptions = {},
): string {
  return path.join(
    options.stateDir ?? resolveStateDir(options.env ?? process.env),
    "logs",
    "stability",
  );
}

function buildBundlePath(dir: string, now: Date, reason: string): string {
  return path.join(
    dir,
    `${BUNDLE_PREFIX}${formatDiagnosticFilenameTimestamp(now)}-${process.pid}-${normalizeReason(reason)}${BUNDLE_SUFFIX}`,
  );
}

function isBundleFile(name: string): boolean {
  return name.startsWith(BUNDLE_PREFIX) && name.endsWith(BUNDLE_SUFFIX);
}

function readObject(value: unknown, label: string): Record<string, unknown> {
  if (!value || typeof value !== "object" || Array.isArray(value)) {
    throw new Error(`Invalid stability bundle: ${label} must be an object`);
  }
  return value as Record<string, unknown>;
}

function readRequiredNumber(value: unknown, label: string): number {
  if (typeof value !== "number" || !Number.isFinite(value)) {
    throw new Error(`Invalid stability bundle: ${label} must be a finite number`);
  }
  return value;
}

function readOptionalPositiveInteger(value: unknown, label: string): number | undefined {
  if (value === undefined) {
    return undefined;
  }
  const parsed = readRequiredNumber(value, label);
  return parsed >= 0 ? Math.floor(parsed) : undefined;
}

function readTimestampMs(value: unknown, label: string): number {
  const timestamp = readRequiredNumber(value, label);
  if (Number.isNaN(new Date(timestamp).getTime())) {
    throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`);
  }
  return timestamp;
}

function readOptionalNumber(value: unknown, label: string): number | undefined {
  if (value === undefined) {
    return undefined;
  }
  return readRequiredNumber(value, label);
}

function readRequiredString(value: unknown, label: string): string {
  if (typeof value !== "string") {
    throw new Error(`Invalid stability bundle: ${label} must be a string`);
  }
  return value;
}

function readTimestampString(value: unknown, label: string): string {
  const timestamp = readRequiredString(value, label);
  if (Number.isNaN(new Date(timestamp).getTime())) {
    throw new Error(`Invalid stability bundle: ${label} must be a valid timestamp`);
  }
  return timestamp;
}

function readCodeString(value: unknown, label: string): string {
  const code = readRequiredString(value, label);
  if (!SAFE_REASON_CODE.test(code)) {
    throw new Error(`Invalid stability bundle: ${label} must be a safe diagnostic code`);
  }
  return code;
}

function readOptionalCodeString(value: unknown, label: string): string | undefined {
  if (value === undefined) {
    return undefined;
  }
  const code = readRequiredString(value, label);
  return SAFE_REASON_CODE.test(code) ? code : undefined;
}

function assignOptionalFields<T extends object>(
  target: T,
  source: Record<string, unknown>,
  label: string,
  fields: readonly (keyof T & string)[],
  read: (value: unknown, label: string) => string | number | undefined,
): void {
  // The fixed order preserves serialized fields and the first failing validation label.
  for (const key of fields) {
    const parsed = read(source[key], `${label}.${key}`);
    if (parsed !== undefined) {
      (target as Record<string, unknown>)[key] = parsed;
    }
  }
}

function readMemoryUsage(value: unknown, label: string): DiagnosticMemoryUsage {
  const memory = readObject(value, label);
  return {
    rssBytes: readRequiredNumber(memory.rssBytes, `${label}.rssBytes`),
    heapTotalBytes: readRequiredNumber(memory.heapTotalBytes, `${label}.heapTotalBytes`),
    heapUsedBytes: readRequiredNumber(memory.heapUsedBytes, `${label}.heapUsedBytes`),
    externalBytes: readRequiredNumber(memory.externalBytes, `${label}.externalBytes`),
    arrayBuffersBytes: readRequiredNumber(memory.arrayBuffersBytes, `${label}.arrayBuffersBytes`),
  };
}

function readHeapStatistics(value: unknown): DiagnosticHeapStatisticsSummary | undefined {
  if (value === undefined) {
    return undefined;
  }
  const source = readObject(value, "evidence.memoryPressure.heapStatistics");
  const result = {} as DiagnosticHeapStatisticsSummary;
  assignOptionalFields(
    result,
    source,
    "evidence.memoryPressure.heapStatistics",
    [
      "totalHeapSizeBytes",
      "totalHeapSizeExecutableBytes",
      "totalPhysicalSizeBytes",
      "totalAvailableSizeBytes",
      "usedHeapSizeBytes",
      "heapSizeLimitBytes",
      "mallocedMemoryBytes",
      "externalMemoryBytes",
    ],
    readOptionalPositiveInteger,
  );
  return Object.keys(result).length > 0 ? result : undefined;
}

function readHeapSpaces(value: unknown): DiagnosticHeapSpaceSummary[] | undefined {
  if (value === undefined) {
    return undefined;
  }
  if (!Array.isArray(value)) {
    throw new Error(
      "Invalid stability bundle: evidence.memoryPressure.heapSpaces must be an array",
    );
  }
  const spaces: DiagnosticHeapSpaceSummary[] = [];
  for (const [index, entry] of value.entries()) {
    const source = readObject(entry, `evidence.memoryPressure.heapSpaces[${index}]`);
    const spaceName = readOptionalCodeString(
      source.spaceName,
      `evidence.memoryPressure.heapSpaces[${index}].spaceName`,
    );
    if (!spaceName) {
      continue;
    }
    spaces.push({
      spaceName,
      spaceSizeBytes:
        readOptionalPositiveInteger(
          source.spaceSizeBytes,
          `evidence.memoryPressure.heapSpaces[${index}].spaceSizeBytes`,
        ) ?? 0,
      spaceUsedBytes:
        readOptionalPositiveInteger(
          source.spaceUsedBytes,
          `evidence.memoryPressure.heapSpaces[${index}].spaceUsedBytes`,
        ) ?? 0,
      spaceAvailableBytes:
        readOptionalPositiveInteger(
          source.spaceAvailableBytes,
          `evidence.memoryPressure.heapSpaces[${index}].spaceAvailableBytes`,
        ) ?? 0,
      physicalSpaceSizeBytes:
        readOptionalPositiveInteger(
          source.physicalSpaceSizeBytes,
          `evidence.memoryPressure.heapSpaces[${index}].physicalSpaceSizeBytes`,
        ) ?? 0,
    });
  }
  return spaces.length > 0 ? spaces : undefined;
}

function readCgroupMemorySummary(value: unknown): DiagnosticCgroupMemorySummary | undefined {
  if (value === undefined) {
    return undefined;
  }
  const source = readObject(value, "evidence.memoryPressure.cgroup");
  const version = readCodeString(
    source.version,
    "evidence.memoryPressure.cgroup.version",
  ) as DiagnosticCgroupMemorySummary["version"];
  if (version !== "v2") {
    return undefined;
  }
  const valuesSource = readObject(source.values, "evidence.memoryPressure.cgroup.values");
  const values: Record<string, number | "max"> = {};
  for (const [key, raw] of Object.entries(valuesSource)) {
    if (!SAFE_REASON_CODE.test(key)) {
      continue;
    }
    if (raw === "max") {
      values[key] = "max";
    } else {
      values[key] =
        readOptionalPositiveInteger(raw, `evidence.memoryPressure.cgroup.values.${key}`) ?? 0;
    }
  }
  return {
    version,
    values,
    events: readNumberMap(source.events, "evidence.memoryPressure.cgroup.events"),
  };
}

function readActiveResources(value: unknown): DiagnosticActiveResourceSummary | undefined {
  if (value === undefined) {
    return undefined;
  }
  const source = readObject(value, "evidence.memoryPressure.activeResources");
  return {
    total:
      readOptionalPositiveInteger(source.total, "evidence.memoryPressure.activeResources.total") ??
      0,
    byType: readNumberMap(source.byType, "evidence.memoryPressure.activeResources.byType"),
  };
}

function readSessionFiles(value: unknown): DiagnosticSessionFileSummary[] | undefined {
  if (value === undefined) {
    return undefined;
  }
  if (!Array.isArray(value)) {
    throw new Error(
      "Invalid stability bundle: evidence.memoryPressure.topSessionFiles must be an array",
    );
  }
  const files: DiagnosticSessionFileSummary[] = [];
  for (const [index, entry] of value.entries()) {
    const source = readObject(entry, `evidence.memoryPressure.topSessionFiles[${index}]`);
    const relativePath = readRequiredString(
      source.relativePath,
      `evidence.memoryPressure.topSessionFiles[${index}].relativePath`,
    );
    if (
      path.isAbsolute(relativePath) ||
      relativePath.includes("..") ||
      relativePath.length > 300 ||
      /[\r\n]/u.test(relativePath)
    ) {
      continue;
    }
    files.push({
      relativePath: sanitizeSessionEvidencePath(relativePath),
      sizeBytes:
        readOptionalPositiveInteger(
          source.sizeBytes,
          `evidence.memoryPressure.topSessionFiles[${index}].sizeBytes`,
        ) ?? 0,
      mtimeMs:
        readOptionalPositiveInteger(
          source.mtimeMs,
          `evidence.memoryPressure.topSessionFiles[${index}].mtimeMs`,
        ) ?? 0,
    });
  }
  return files.length > 0 ? files : undefined;
}

function readMemoryPressureEvidence(
  value: unknown,
): DiagnosticMemoryPressureBundleEvidence | undefined {
  if (value === undefined) {
    return undefined;
  }
  const pressure = readObject(value, "evidence.memoryPressure");
  const level = readCodeString(
    pressure.level,
    "evidence.memoryPressure.level",
  ) as DiagnosticMemoryPressureEvent["level"];
  const reason = readCodeString(
    pressure.reason,
    "evidence.memoryPressure.reason",
  ) as DiagnosticMemoryPressureEvent["reason"];
  if ((level !== "warning" && level !== "critical") || !isMemoryPressureReason(reason)) {
    return undefined;
  }
  const heapStatistics = readHeapStatistics(pressure.heapStatistics);
  const heapSpaces = readHeapSpaces(pressure.heapSpaces);
  const cgroup = readCgroupMemorySummary(pressure.cgroup);
  const activeResources = readActiveResources(pressure.activeResources);
  const topSessionFiles = readSessionFiles(pressure.topSessionFiles);
  const result: DiagnosticMemoryPressureBundleEvidence = {
    level,
    reason,
    memory: readMemoryUsage(pressure.memory, "evidence.memoryPressure.memory"),
  };
  assignOptionalFields(
    result,
    pressure,
    "evidence.memoryPressure",
    ["thresholdBytes", "rssGrowthBytes", "windowMs"],
    readOptionalNumber,
  );
  return {
    ...result,
    ...(heapStatistics ? { heapStatistics } : {}),
    ...(heapSpaces ? { heapSpaces } : {}),
    ...(cgroup ? { cgroup } : {}),
    ...(activeResources ? { activeResources } : {}),
    ...(topSessionFiles ? { topSessionFiles } : {}),
  };
}

function readBundleEvidence(value: unknown): DiagnosticStabilityBundleEvidence | undefined {
  if (value === undefined) {
    return undefined;
  }
  const source = readObject(value, "evidence");
  const memoryPressure = readMemoryPressureEvidence(source.memoryPressure);
  let shutdown: DiagnosticStabilityBundleEvidence["shutdown"];
  if (source.shutdown !== undefined) {
    const shutdownSource = readObject(source.shutdown, "evidence.shutdown");
    if (!Array.isArray(shutdownSource.errors)) {
      throw new Error("Invalid stability bundle: evidence.shutdown.errors must be an array");
    }
    shutdown = {
      step: readCodeString(shutdownSource.step, "evidence.shutdown.step"),
      errors: shutdownSource.errors.slice(0, MAX_SHUTDOWN_ERRORS).map(readShutdownError),
    };
  }
  return memoryPressure || shutdown
    ? { ...(memoryPressure ? { memoryPressure } : {}), ...(shutdown ? { shutdown } : {}) }
    : undefined;
}

function readNumberMap(value: unknown, label: string): Record<string, number> {
  const source = readObject(value, label);
  const result: Record<string, number> = {};
  for (const [key, entry] of Object.entries(source)) {
    if (!SAFE_REASON_CODE.test(key)) {
      continue;
    }
    result[key] = readRequiredNumber(entry, `${label}.${key}`);
  }
  return result;
}

function readOptionalMemorySummary(
  value: unknown,
): DiagnosticStabilitySnapshot["summary"]["memory"] | undefined {
  if (value === undefined) {
    return undefined;
  }
  const memory = readObject(value, "snapshot.summary.memory");
  const latest =
    memory.latest === undefined
      ? undefined
      : readMemoryUsage(memory.latest, "snapshot.summary.memory.latest");
  return {
    ...(latest ? { latest } : {}),
    ...(memory.maxRssBytes !== undefined
      ? {
          maxRssBytes: readRequiredNumber(
            memory.maxRssBytes,
            "snapshot.summary.memory.maxRssBytes",
          ),
        }
      : {}),
    ...(memory.maxHeapUsedBytes !== undefined
      ? {
          maxHeapUsedBytes: readRequiredNumber(
            memory.maxHeapUsedBytes,
            "snapshot.summary.memory.maxHeapUsedBytes",
          ),
        }
      : {}),
    pressureCount: readRequiredNumber(
      memory.pressureCount,
      "snapshot.summary.memory.pressureCount",
    ),
  };
}

function readOptionalPayloadLargeSummary(
  value: unknown,
): DiagnosticStabilitySnapshot["summary"]["payloadLarge"] | undefined {
  if (value === undefined) {
    return undefined;
  }
  const payloadLarge = readObject(value, "snapshot.summary.payloadLarge");
  return {
    count: readRequiredNumber(payloadLarge.count, "snapshot.summary.payloadLarge.count"),
    rejected: readRequiredNumber(payloadLarge.rejected, "snapshot.summary.payloadLarge.rejected"),
    truncated: readRequiredNumber(
      payloadLarge.truncated,
      "snapshot.summary.payloadLarge.truncated",
    ),
    chunked: readRequiredNumber(payloadLarge.chunked, "snapshot.summary.payloadLarge.chunked"),
    bySurface: readNumberMap(payloadLarge.bySurface, "snapshot.summary.payloadLarge.bySurface"),
  };
}

function readStabilityEventRecord(
  value: unknown,
  label: string,
): DiagnosticStabilitySnapshot["events"][number] {
  const record = readObject(value, label);
  const sanitized: DiagnosticStabilitySnapshot["events"][number] = {
    seq: readRequiredNumber(record.seq, `${label}.seq`),
    ts: readTimestampMs(record.ts, `${label}.ts`),
    type: readCodeString(
      record.type,
      `${label}.type`,
    ) as DiagnosticStabilitySnapshot["events"][number]["type"],
  };

  assignOptionalFields(
    sanitized,
    record,
    label,
    [
      "channel",
      "pluginId",
      "source",
      "surface",
      "action",
      "reason",
      "outcome",
      "level",
      "phase",
      "approvalId",
      "detector",
      "toolName",
      "activeWorkKind",
      "pairedToolName",
      "provider",
      "model",
    ],
    readOptionalCodeString,
  );

  assignOptionalFields(
    sanitized,
    record,
    label,
    [
      "durationMs",
      "requestBytes",
      "responseBytes",
      "timeToFirstByteMs",
      "costUsd",
      "count",
      "bytes",
      "limitBytes",
      "thresholdBytes",
      "rssGrowthBytes",
      "windowMs",
      "ageMs",
      "queueDepth",
      "queueSize",
      "queueLength",
      "waitMs",
      "active",
      "waiting",
      "queued",
      "droppedEvents",
      "droppedTrustedEvents",
      "droppedUntrustedEvents",
      "droppedPriorityEvents",
      "maxQueueLength",
      "drainBatchSize",
    ],
    readOptionalNumber,
  );

  if (record.webhooks !== undefined) {
    const webhooks = readObject(record.webhooks, `${label}.webhooks`);
    sanitized.webhooks = {
      received: readRequiredNumber(webhooks.received, `${label}.webhooks.received`),
      processed: readRequiredNumber(webhooks.processed, `${label}.webhooks.processed`),
      errors: readRequiredNumber(webhooks.errors, `${label}.webhooks.errors`),
    };
  }
  if (record.memory !== undefined) {
    sanitized.memory = readMemoryUsage(record.memory, `${label}.memory`);
  }
  if (record.usage !== undefined) {
    const usage = readObject(record.usage, `${label}.usage`);
    sanitized.usage = {};
    assignOptionalFields(
      sanitized.usage,
      usage,
      `${label}.usage`,
      ["input", "output", "cacheRead", "cacheWrite", "promptTokens", "total"],
      readOptionalNumber,
    );
  }
  if (record.context !== undefined) {
    const context = readObject(record.context, `${label}.context`);
    sanitized.context = {};
    assignOptionalFields(
      sanitized.context,
      context,
      `${label}.context`,
      ["limit", "used"],
      readOptionalNumber,
    );
  }

  return sanitized;
}

function readStabilitySnapshot(value: unknown): DiagnosticStabilitySnapshot {
  const snapshot = readObject(value, "snapshot");
  const generatedAt = readTimestampString(snapshot.generatedAt, "snapshot.generatedAt");
  const capacity = readRequiredNumber(snapshot.capacity, "snapshot.capacity");
  const count = readRequiredNumber(snapshot.count, "snapshot.count");
  const dropped = readRequiredNumber(snapshot.dropped, "snapshot.dropped");
  const firstSeq = readOptionalNumber(snapshot.firstSeq, "snapshot.firstSeq");
  const lastSeq = readOptionalNumber(snapshot.lastSeq, "snapshot.lastSeq");
  if (!Array.isArray(snapshot.events)) {
    throw new Error("Invalid stability bundle: snapshot.events must be an array");
  }
  const events = snapshot.events.map((event, index) =>
    readStabilityEventRecord(event, `snapshot.events[${index}]`),
  );
  const summary = readObject(snapshot.summary, "snapshot.summary");
  return {
    generatedAt,
    capacity,
    count,
    dropped,
    ...(firstSeq !== undefined ? { firstSeq } : {}),
    ...(lastSeq !== undefined ? { lastSeq } : {}),
    events,
    summary: {
      byType: readNumberMap(summary.byType, "snapshot.summary.byType"),
      ...(summary.memory !== undefined
        ? { memory: readOptionalMemorySummary(summary.memory) }
        : {}),
      ...(summary.payloadLarge !== undefined
        ? { payloadLarge: readOptionalPayloadLargeSummary(summary.payloadLarge) }
        : {}),
    },
  };
}

function parseDiagnosticStabilityBundle(value: unknown): DiagnosticStabilityBundle {
  const bundle = readObject(value, "bundle");
  if (bundle.version !== DIAGNOSTIC_STABILITY_BUNDLE_VERSION) {
    throw new Error(`Unsupported stability bundle version: ${String(bundle.version)}`);
  }
  const processInfo = readObject(bundle.process, "process");
  readObject(bundle.host, "host");
  const error = bundle.error === undefined ? undefined : readSafeErrorMetadata(bundle.error);
  const evidence = readBundleEvidence(bundle.evidence);
  return {
    version: DIAGNOSTIC_STABILITY_BUNDLE_VERSION,
    generatedAt: readTimestampString(bundle.generatedAt, "generatedAt"),
    reason: normalizeReason(readRequiredString(bundle.reason, "reason")),
    process: {
      pid: readRequiredNumber(processInfo.pid, "process.pid"),
      platform: readCodeString(processInfo.platform, "process.platform") as NodeJS.Platform,
      arch: readCodeString(processInfo.arch, "process.arch"),
      node: readCodeString(processInfo.node, "process.node"),
      uptimeMs: readRequiredNumber(processInfo.uptimeMs, "process.uptimeMs"),
    },
    host: {
      hostname: REDACTED_HOSTNAME,
    },
    ...(error ? { error } : {}),
    ...(evidence ? { evidence } : {}),
    snapshot: readStabilitySnapshot(bundle.snapshot),
  };
}

function sanitizeSessionEvidencePath(relativePath: string): string {
  const parts = relativePath.split("/");
  if (parts.length === 4 && parts[0] === "agents" && parts[2] === "sessions") {
    return `agents/<agent>/sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[3], "parts entry at 3"))}`;
  }
  if (parts.length === 2 && parts[0] === "sessions") {
    return `sessions/${sanitizeSessionEvidenceFileName(expectDefined(parts[1], "parts entry at 1"))}`;
  }
  return redactSensitiveText(relativePath, { mode: "tools" });
}

function sanitizeSessionEvidenceFileName(fileName: string): string {
  if (fileName === "sessions.json") {
    return "sessions.json";
  }
  if (fileName.endsWith(".jsonl")) {
    return "<session>.jsonl";
  }
  if (fileName.endsWith(".json")) {
    return "<session>.json";
  }
  return "<session>";
}

function isMemoryPressureReason(reason: string): reason is DiagnosticMemoryPressureEvent["reason"] {
  return reason === "rss_threshold" || reason === "heap_threshold" || reason === "rss_growth";
}

function listDiagnosticStabilityBundleFilesSync(
  options: DiagnosticStabilityBundleLocationOptions = {},
): DiagnosticStabilityBundleFile[] {
  const dir = resolveDiagnosticStabilityBundleDir(options);
  try {
    return fs
      .readdirSync(dir, { withFileTypes: true })
      .filter((entry) => entry.isFile() && isBundleFile(entry.name))
      .map((entry) => {
        const file = path.join(dir, entry.name);
        return {
          path: file,
          mtimeMs: fs.statSync(file).mtimeMs,
        };
      })
      .toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.path.localeCompare(a.path));
  } catch (error) {
    if (isMissingPathError(error)) {
      return [];
    }
    throw error;
  }
}

export function readDiagnosticStabilityBundleFileSync(
  file: string,
): ReadDiagnosticStabilityBundleResult {
  try {
    const stat = fs.statSync(file);
    if (stat.size > MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES) {
      throw new Error(
        `Stability bundle is too large: ${stat.size} bytes exceeds ${MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES}`,
      );
    }
    const raw = fs.readFileSync(file, "utf8");
    const bundle = parseDiagnosticStabilityBundle(JSON.parse(raw));
    return {
      status: "found",
      path: file,
      mtimeMs: stat.mtimeMs,
      bundle,
    };
  } catch (error) {
    return { status: "failed", path: file, error };
  }
}

export function readLatestDiagnosticStabilityBundleSync(
  options: DiagnosticStabilityBundleLocationOptions = {},
): ReadDiagnosticStabilityBundleResult {
  try {
    const latest = listDiagnosticStabilityBundleFilesSync(options)[0];
    if (!latest) {
      return {
        status: "missing",
        dir: resolveDiagnosticStabilityBundleDir(options),
      };
    }
    return readDiagnosticStabilityBundleFileSync(latest.path);
  } catch (error) {
    return { status: "failed", error };
  }
}

function pruneOldBundles(dir: string, retention: number, retainedFile: string): void {
  if (!Number.isFinite(retention) || retention < 1) {
    return;
  }
  try {
    const entries = fs
      .readdirSync(dir, { withFileTypes: true })
      .filter((entry) => entry.isFile() && isBundleFile(entry.name))
      .map((entry) => {
        const file = path.join(dir, entry.name);
        let mtimeMs = 0;
        try {
          mtimeMs = fs.statSync(file).mtimeMs;
        } catch {
          // Missing files are ignored below.
        }
        return { file, mtimeMs };
      })
      .filter((entry) => entry.file !== retainedFile)
      .toSorted((a, b) => b.mtimeMs - a.mtimeMs || b.file.localeCompare(a.file));

    for (const entry of entries.slice(retention - 1)) {
      try {
        fs.unlinkSync(entry.file);
      } catch {
        // Retention cleanup must not block failure handling.
      }
    }
  } catch {
    // Retention cleanup must not block failure handling.
  }
}

export function writeDiagnosticStabilityBundleSync(
  options: WriteDiagnosticStabilityBundleOptions,
): WriteDiagnosticStabilityBundleResult {
  try {
    const now = options.now ?? new Date();
    const snapshot = getDiagnosticStabilitySnapshot({
      limit: options.limit ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_LIMIT,
    });
    if (!options.includeEmpty && snapshot.count === 0) {
      return { status: "skipped", reason: "empty" };
    }

    const reason = normalizeReason(options.reason);
    const error = options.error ? readSafeErrorMetadata(options.error) : undefined;
    const evidence = options.shutdownStep
      ? {
          ...options.evidence,
          shutdown: {
            step: readCodeString(options.shutdownStep, "shutdownStep"),
            errors: collectShutdownErrors(options.error),
          },
        }
      : options.evidence;
    const bundle: DiagnosticStabilityBundle = {
      version: DIAGNOSTIC_STABILITY_BUNDLE_VERSION,
      generatedAt: now.toISOString(),
      reason,
      process: {
        pid: process.pid,
        platform: process.platform,
        arch: process.arch,
        node: process.versions.node,
        uptimeMs: Math.round(process.uptime() * 1000),
      },
      host: {
        hostname: REDACTED_HOSTNAME,
      },
      ...(error ? { error } : {}),
      ...(evidence ? { evidence } : {}),
      snapshot,
    };

    const dir = resolveDiagnosticStabilityBundleDir(options);
    const file = buildBundlePath(dir, now, reason);
    replaceFileAtomicSync({
      filePath: file,
      content: `${JSON.stringify(bundle, null, 2)}\n`,
      dirMode: 0o700,
      mode: 0o600,
      tempPrefix: ".openclaw-stability",
    });
    pruneOldBundles(dir, options.retention ?? DEFAULT_DIAGNOSTIC_STABILITY_BUNDLE_RETENTION, file);
    return { status: "written", path: file, bundle };
  } catch (error) {
    return { status: "failed", error };
  }
}

export function writeDiagnosticStabilityBundleForFailureSync(
  reason: string,
  error?: unknown,
  options: WriteDiagnosticStabilityBundleForFailureOptions = {},
): DiagnosticStabilityBundleFailureWriteOutcome {
  const result = writeDiagnosticStabilityBundleSync({
    ...options,
    reason,
    error,
    includeEmpty: true,
  });
  if (result.status === "written") {
    return {
      status: "written",
      path: result.path,
      message: `wrote stability bundle: ${result.path}`,
    };
  }
  if (result.status === "failed") {
    return {
      status: "failed",
      error: result.error,
      message: `failed to write stability bundle: ${String(result.error)}`,
    };
  }
  return result;
}

export function installDiagnosticStabilityFatalHook(
  options: WriteDiagnosticStabilityBundleForFailureOptions = {},
): void {
  if (fatalHookUnsubscribe) {
    return;
  }
  fatalHookUnsubscribe = registerFatalErrorHook(({ reason, error }) => {
    const result = writeDiagnosticStabilityBundleForFailureSync(reason, error, options);
    return "message" in result ? result.message : undefined;
  });
}

export function uninstallDiagnosticStabilityFatalHook(): void {
  fatalHookUnsubscribe?.();
  fatalHookUnsubscribe = null;
}

/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */