openclaw / src /logging /diagnostic-stability-bundle.test.ts
SaylorTwift's picture
SaylorTwift HF Staff
Add files using upload-large-folder tool
76289e7 verified
Raw History Blame Contribute Delete
23.1 kB
// Diagnostic stability bundle tests cover stable diagnostic bundle generation.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { emitDiagnosticEvent, resetDiagnosticEventsForTest } from "../infra/diagnostic-events.js";
import { registerFatalErrorHook, runFatalErrorHooks } from "../infra/fatal-error-hooks.js";
import {
installDiagnosticStabilityFatalHook,
MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES,
readDiagnosticStabilityBundleFileSync,
readLatestDiagnosticStabilityBundleSync,
uninstallDiagnosticStabilityFatalHook,
writeDiagnosticStabilityBundleForFailureSync,
writeDiagnosticStabilityBundleSync,
type DiagnosticStabilityBundle,
} from "./diagnostic-stability-bundle.js";
import {
resetDiagnosticStabilityRecorderForTest,
startDiagnosticStabilityRecorder,
stopDiagnosticStabilityRecorder,
} from "./diagnostic-stability.js";
describe("diagnostic stability bundles", () => {
let tempDir: string;
function resetStabilityBundleTestState(): void {
resetDiagnosticEventsForTest();
resetDiagnosticStabilityRecorderForTest();
uninstallDiagnosticStabilityFatalHook();
}
beforeEach(() => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-stability-bundle-"));
resetStabilityBundleTestState();
});
afterEach(() => {
stopDiagnosticStabilityRecorder();
resetStabilityBundleTestState();
fs.rmSync(tempDir, { recursive: true, force: true });
});
function readBundle(file: string): DiagnosticStabilityBundle {
return JSON.parse(fs.readFileSync(file, "utf8")) as DiagnosticStabilityBundle;
}
function createImportedBundle(): Record<string, unknown> {
return {
version: 1,
generatedAt: "2026-04-22T12:00:00.000Z",
reason: "gateway.restart_startup_failed",
process: {
pid: 123,
platform: "darwin",
arch: "arm64",
node: "24.14.1",
uptimeMs: 1000,
},
host: {
hostname: "<redacted-hostname>",
},
snapshot: {
generatedAt: "2026-04-22T12:00:00.000Z",
capacity: 1000,
count: 1,
dropped: 0,
events: [{ seq: 1, ts: 1, type: "webhook.received" }],
summary: { byType: { "webhook.received": 1 } },
},
};
}
it("writes a payload-free bundle with safe failure metadata", () => {
startDiagnosticStabilityRecorder();
emitDiagnosticEvent({
type: "webhook.error",
channel: "telegram",
chatId: "chat-secret",
error: "raw diagnostic error with message body",
});
emitDiagnosticEvent({
type: "payload.large",
surface: "gateway.http.json",
action: "rejected",
bytes: 2048,
limitBytes: 1024,
reason: "json_body_limit",
});
const secret = "sk-1234567890abcdef";
const error = Object.assign(
new Error(
`Startup failed: OPENAI_API_KEY=${secret} while opening google/web-search-contract-api.js`,
),
{ code: "ERR_TEST" },
);
const result = writeDiagnosticStabilityBundleSync({
reason: "gateway.restart_startup_failed",
error,
stateDir: tempDir,
now: new Date("2026-04-22T12:00:00.000Z"),
});
expect(result.status).toBe("written");
const file = result.status === "written" ? result.path : "";
const bundle = readBundle(file);
const raw = fs.readFileSync(file, "utf8");
expect(bundle.version).toBe(1);
expect(bundle.generatedAt).toBe("2026-04-22T12:00:00.000Z");
expect(bundle.reason).toBe("gateway.restart_startup_failed");
expect(bundle.error?.name).toBe("Error");
expect(bundle.error?.code).toBe("ERR_TEST");
expect(bundle.host.hostname).toBe("<redacted-hostname>");
expect(bundle.snapshot.count).toBe(2);
expect(bundle.snapshot.events[0]?.type).toBe("webhook.error");
expect(bundle.snapshot.events[0]?.channel).toBe("telegram");
expect(bundle.snapshot.events[0]).not.toHaveProperty("chatId");
expect(bundle.snapshot.events[0]).not.toHaveProperty("error");
expect(bundle.error?.message).toContain("google/web-search-contract-api.js");
expect(bundle.error?.message).not.toContain(secret);
expect(raw).not.toContain("chat-secret");
expect(raw).not.toContain("message body");
expect(raw).not.toContain(secret);
expect(raw).not.toContain(os.hostname());
});
it("skips empty recorder snapshots by default", () => {
const result = writeDiagnosticStabilityBundleSync({
reason: "uncaught_exception",
stateDir: tempDir,
});
expect(result).toEqual({ status: "skipped", reason: "empty" });
expect(fs.existsSync(path.join(tempDir, "logs", "stability"))).toBe(false);
});
it("writes failure bundles even when the recorder snapshot is empty", () => {
const result = writeDiagnosticStabilityBundleForFailureSync(
"gateway.restart_startup_failed",
Object.assign(new Error("raw startup config payload"), { code: "ERR_CONFIG_PARSE" }),
{
stateDir: tempDir,
now: new Date("2026-04-22T12:00:00.000Z"),
},
);
if (result.status !== "written") {
throw new Error(`expected written bundle, got ${result.status}`);
}
const bundle = readBundle(result.path);
const raw = fs.readFileSync(result.path, "utf8");
expect(bundle.reason).toBe("gateway.restart_startup_failed");
expect(bundle.error).toEqual({
name: "Error",
code: "ERR_CONFIG_PARSE",
message: "raw startup config payload",
});
expect(bundle.snapshot.count).toBe(0);
expect(bundle.snapshot.events).toEqual([]);
expect(raw).not.toContain("stack");
});
it("keeps bounded failure messages UTF-16 safe", () => {
const prefix = "a".repeat(499);
const result = writeDiagnosticStabilityBundleForFailureSync(
"gateway.restart_startup_failed",
new Error(`${prefix}😀${"b".repeat(500)}`),
{ stateDir: tempDir },
);
expect(result.status).toBe("written");
if (result.status !== "written") {
return;
}
expect(readBundle(result.path).error?.message).toBe(`${prefix}...`);
});
it("preserves redacted shutdown causes and stacks through bundle readback", () => {
const secret = "sk-1234567890abcdef";
const cause = new TypeError(`fixture close failed: OPENAI_API_KEY=${secret}`);
cause.stack = `${cause.name}: ${cause.message}\n at closeOwner (fixture.js:12:3)`;
const stepError = new Error("shutdown step failed (gateway lifetime sidecars)", { cause });
const error = new AggregateError([stepError], "Gateway shutdown did not complete cleanly");
const result = writeDiagnosticStabilityBundleForFailureSync(
"gateway.restart_close_failed",
error,
{
stateDir: tempDir,
shutdownStep: "gateway-server-close",
},
);
if (result.status !== "written") {
throw new Error(`expected written bundle, got ${result.status}`);
}
const readback = readDiagnosticStabilityBundleFileSync(result.path);
expect(readback.status).toBe("found");
if (readback.status !== "found") {
return;
}
expect(readback.bundle.evidence?.shutdown).toMatchObject({
step: "gateway-server-close",
errors: [
{ name: "AggregateError", message: "Gateway shutdown did not complete cleanly" },
{ name: "Error", message: "shutdown step failed (gateway lifetime sidecars)" },
{
name: "TypeError",
message: expect.stringContaining("fixture close failed"),
stack: expect.stringContaining("at closeOwner (fixture.js:12:3)"),
},
],
});
expect(fs.readFileSync(result.path, "utf8")).not.toContain(secret);
expect(JSON.stringify(readback.bundle)).not.toContain(secret);
});
it("bounds cyclic shutdown error graphs and stack text without losing non-Error throws", () => {
const error = new AggregateError(
Array.from({ length: 100 }, (_, index) => new Error(`failure ${index}`)),
"shutdown failures",
);
error.cause = error;
error.stack = "a".repeat(7_999) + "😀" + "b".repeat(1_000);
const result = writeDiagnosticStabilityBundleSync({
reason: "gateway.stop_close_failed",
error,
shutdownStep: "gateway-server-close",
stateDir: tempDir,
includeEmpty: true,
});
expect(result.status).toBe("written");
if (result.status !== "written") {
return;
}
expect(result.bundle.evidence?.shutdown?.errors.length).toBeLessThanOrEqual(32);
expect(result.bundle.evidence?.shutdown?.errors[0]?.stack).toBe("a".repeat(7_999));
const nonError = writeDiagnosticStabilityBundleSync({
reason: "gateway.stop_close_failed",
error: "fixture string failure",
shutdownStep: "startup-operations",
stateDir: tempDir,
includeEmpty: true,
});
expect(nonError.status === "written" && nonError.bundle.evidence?.shutdown).toEqual({
step: "startup-operations",
errors: [{ message: "fixture string failure" }],
});
});
it("registers a fatal hook only while installed", () => {
startDiagnosticStabilityRecorder();
emitDiagnosticEvent({ type: "webhook.received", channel: "telegram" });
installDiagnosticStabilityFatalHook({ stateDir: tempDir });
const messages = runFatalErrorHooks({
reason: "fatal_unhandled_rejection",
error: Object.assign(new Error("raw text"), { code: "ERR_OUT_OF_MEMORY" }),
});
expect(messages).toHaveLength(1);
expect(messages[0]).toContain("wrote stability bundle:");
expect(messages[0]).toContain(tempDir);
uninstallDiagnosticStabilityFatalHook();
expect(runFatalErrorHooks({ reason: "uncaught_exception" })).toStrictEqual([]);
const unsubscribeIndependent = registerFatalErrorHook(() => "independent diagnostic");
try {
const reinstalledDir = path.join(tempDir, "reinstalled");
installDiagnosticStabilityFatalHook({ stateDir: reinstalledDir });
const reinstalledMessages = runFatalErrorHooks({ reason: "uncaught_exception" });
expect(reinstalledMessages).toHaveLength(2);
expect(reinstalledMessages[0]).toBe("independent diagnostic");
expect(reinstalledMessages[1]).toContain("wrote stability bundle:");
expect(reinstalledMessages[1]).toContain(reinstalledDir);
uninstallDiagnosticStabilityFatalHook();
expect(runFatalErrorHooks({ reason: "uncaught_exception" })).toEqual([
"independent diagnostic",
]);
uninstallDiagnosticStabilityFatalHook();
expect(runFatalErrorHooks({ reason: "uncaught_exception" })).toEqual([
"independent diagnostic",
]);
} finally {
uninstallDiagnosticStabilityFatalHook();
unsubscribeIndependent();
}
});
it("retains only the newest bundle files", () => {
startDiagnosticStabilityRecorder();
emitDiagnosticEvent({ type: "webhook.received", channel: "telegram" });
for (let index = 0; index < 4; index += 1) {
const result = writeDiagnosticStabilityBundleSync({
reason: "gateway.restart_respawn_failed",
stateDir: tempDir,
now: new Date(`2026-04-22T12:00:0${index}.000Z`),
retention: 2,
});
expect(result.status).toBe("written");
}
const bundleDir = path.join(tempDir, "logs", "stability");
const files = fs.readdirSync(bundleDir).toSorted();
expect(files).toHaveLength(2);
expect(files[0]).toContain("12-00-02");
expect(files[1]).toContain("12-00-03");
});
it.each([1, 2])(
"keeps the published bundle within retention %i despite future mtimes",
(retention) => {
for (let index = 0; index < retention; index++) {
const older = writeDiagnosticStabilityBundleForFailureSync(
"gateway.startup_failed",
undefined,
{
stateDir: tempDir,
retention,
now: new Date(Date.UTC(2026, 3, 22, 12, 0, index)),
},
);
expect(older.status).toBe("written");
if (older.status !== "written") {
throw new Error("Fixture publication failed");
}
const future = new Date(Date.UTC(2036, 3, 22, 12, 0, index));
fs.utimesSync(older.path, future, future);
}
const current = writeDiagnosticStabilityBundleForFailureSync(
"gateway.restart_respawn_failed",
undefined,
{
stateDir: tempDir,
retention,
now: new Date("2026-04-22T12:01:00.000Z"),
},
);
expect(current.status).toBe("written");
if (current.status !== "written") {
throw new Error("Current publication failed");
}
expect(current.message).toContain(current.path);
expect(fs.existsSync(current.path)).toBe(true);
expect(readDiagnosticStabilityBundleFileSync(current.path).status).toBe("found");
expect(fs.readdirSync(path.dirname(current.path))).toHaveLength(retention);
},
);
it("reads the newest retained bundle", () => {
startDiagnosticStabilityRecorder();
emitDiagnosticEvent({ type: "webhook.received", channel: "telegram" });
const older = writeDiagnosticStabilityBundleSync({
reason: "gateway.restart_startup_failed",
stateDir: tempDir,
now: new Date("2026-04-22T12:00:00.000Z"),
});
const newer = writeDiagnosticStabilityBundleSync({
reason: "gateway.restart_respawn_failed",
stateDir: tempDir,
now: new Date("2026-04-22T12:00:01.000Z"),
});
expect(older.status).toBe("written");
expect(newer.status).toBe("written");
const latest = readLatestDiagnosticStabilityBundleSync({ stateDir: tempDir });
expect(latest.status).toBe("found");
expect(latest.status === "found" ? latest.path : "").toContain("12-00-01");
expect(latest.status === "found" ? latest.bundle.reason : "").toBe(
"gateway.restart_respawn_failed",
);
});
it("sanitizes imported bundles before returning them", () => {
const file = path.join(tempDir, "imported.json");
const bundle = createImportedBundle();
const retainedRuntimeEvidence = {
heapStatistics: { heapSizeLimitBytes: 8192, usedHeapSizeBytes: 1536 },
heapSpaces: [
{
spaceName: "old_space",
spaceSizeBytes: 2048,
spaceUsedBytes: 1536,
spaceAvailableBytes: 512,
physicalSpaceSizeBytes: 2048,
},
],
cgroup: {
version: "v2",
values: { current: 4096, max: "max" },
events: { high: 2, "events.local.oom": 1 },
},
activeResources: { total: 3, byType: { Timeout: 2, PipeWrap: 1 } },
};
Object.assign(bundle, {
reason: "private reason token=secret",
privateTopLevel: "top-level-secret",
evidence: {
memoryPressure: {
...retainedRuntimeEvidence,
heapStatistics: {
...retainedRuntimeEvidence.heapStatistics,
totalHeapSizeBytes: 1536.75,
},
level: "critical",
reason: "rss_threshold",
thresholdBytes: 0,
rssGrowthBytes: -1,
windowMs: 0.5,
memory: {
rssBytes: 4096,
heapTotalBytes: 2048,
heapUsedBytes: 1536,
externalBytes: 128,
arrayBuffersBytes: 64,
},
topSessionFiles: [
{
relativePath: "agents/main/sessions/raw-secret-session.jsonl",
sizeBytes: 4096,
mtimeMs: 1,
},
],
},
},
error: {
name: "private error name",
code: "ERR_TEST",
message: "OPENAI_API_KEY=sk-1234567890abcdef",
},
});
Object.assign(bundle.process as Record<string, unknown>, {
command: "process-command-secret",
});
Object.assign(bundle.host as Record<string, unknown>, {
hostname: "private-hostname",
fqdn: "host-extra-secret",
});
const snapshot = bundle.snapshot as Record<string, unknown>;
Object.assign(snapshot, {
count: 3,
privateSnapshot: "snapshot-secret",
events: [
{
seq: 1,
ts: 1,
type: "webhook.error",
channel: "telegram",
reason: "private event reason",
chatId: "chat-id-secret",
error: "event-error-secret",
},
{
seq: 2,
ts: 2,
type: "exec.approval.followup_suppressed",
approvalId: "approval-imported-123",
reason: "session_rebound",
phase: "gateway_preflight",
command: "raw command secret",
},
{
seq: 3,
ts: 3,
type: "model.usage",
costUsd: 0,
durationMs: 0,
usage: {},
context: {},
},
],
summary: {
byType: {
"webhook.error": 1,
"exec.approval.followup_suppressed": 1,
"model.usage": 1,
"private summary type": 1,
},
privateSummary: "summary-secret",
},
});
fs.writeFileSync(file, `${JSON.stringify(bundle, null, 2)}\n`, "utf8");
const result = readDiagnosticStabilityBundleFileSync(file);
expect(result.status).toBe("found");
if (result.status !== "found") {
return;
}
expect(result.bundle.reason).toBe("unknown");
expect(result.bundle.host).toEqual({ hostname: "<redacted-hostname>" });
expect(result.bundle.error?.code).toBe("ERR_TEST");
expect(result.bundle.error?.message).toContain("OPENAI_API_KEY=");
expect(result.bundle.error?.message).not.toContain("sk-1234567890abcdef");
expect(result.bundle.evidence?.memoryPressure?.topSessionFiles?.[0]?.relativePath).toBe(
"agents/<agent>/sessions/<session>.jsonl",
);
expect(result.bundle.evidence?.memoryPressure).toMatchObject(retainedRuntimeEvidence);
expect(result.bundle.evidence?.memoryPressure).toMatchObject({
thresholdBytes: 0,
rssGrowthBytes: -1,
windowMs: 0.5,
heapStatistics: { totalHeapSizeBytes: 1536 },
});
expect(Object.keys(result.bundle.evidence?.memoryPressure?.heapStatistics ?? {})).toEqual([
"totalHeapSizeBytes",
"usedHeapSizeBytes",
"heapSizeLimitBytes",
]);
expect(result.bundle.snapshot.events[0]).toEqual({
seq: 1,
ts: 1,
type: "webhook.error",
channel: "telegram",
});
expect(result.bundle.snapshot.events[1]).toEqual({
seq: 2,
ts: 2,
type: "exec.approval.followup_suppressed",
approvalId: "approval-imported-123",
reason: "session_rebound",
phase: "gateway_preflight",
});
expect(JSON.stringify(result.bundle.snapshot.events[2])).toBe(
JSON.stringify({
seq: 3,
ts: 3,
type: "model.usage",
durationMs: 0,
costUsd: 0,
usage: {},
context: {},
}),
);
expect(result.bundle.snapshot.summary.byType).toEqual({
"webhook.error": 1,
"exec.approval.followup_suppressed": 1,
"model.usage": 1,
});
const sanitized = JSON.stringify(result.bundle);
for (const secret of [
"private reason",
"top-level-secret",
"private error name",
"sk-1234567890abcdef",
"process-command-secret",
"private-hostname",
"host-extra-secret",
"snapshot-secret",
"private event reason",
"raw-secret-session",
"chat-id-secret",
"event-error-secret",
"raw command secret",
"private summary type",
"summary-secret",
]) {
expect(sanitized).not.toContain(secret);
}
});
it("rejects malformed bundle files", () => {
const file = path.join(tempDir, "invalid.json");
fs.writeFileSync(file, "{}\n", "utf8");
const result = readDiagnosticStabilityBundleFileSync(file);
expect(result.status).toBe("failed");
expect(result.status === "failed" ? String(result.error) : "").toContain(
"Unsupported stability bundle version",
);
});
it("rejects oversized bundle files before reading them", () => {
const file = path.join(tempDir, "oversized.json");
fs.closeSync(fs.openSync(file, "w"));
fs.truncateSync(file, MAX_DIAGNOSTIC_STABILITY_BUNDLE_BYTES + 1);
const result = readDiagnosticStabilityBundleFileSync(file);
expect(result.status).toBe("failed");
expect(result.status === "failed" ? String(result.error) : "").toContain(
"Stability bundle is too large",
);
});
it("rejects malformed bundle snapshots before returning them", () => {
const baseBundle = createImportedBundle();
const baseSnapshot = baseBundle.snapshot as Record<string, unknown>;
const cases = [
{
name: "malformed-event",
bundle: {
...baseBundle,
snapshot: {
...baseSnapshot,
events: [{ type: "webhook.received", ts: 1 }],
},
},
error: "snapshot.events[0].seq",
},
{
name: "out-of-range-event-timestamp",
bundle: {
...baseBundle,
snapshot: {
...baseSnapshot,
events: [{ seq: 1, ts: 9e15, type: "webhook.received" }],
},
},
error: "snapshot.events[0].ts",
},
{
name: "null-summary",
bundle: {
...baseBundle,
snapshot: {
...baseSnapshot,
summary: null,
},
},
error: "snapshot.summary",
},
{
name: "optional-code-before-number",
bundle: {
...baseBundle,
snapshot: {
...baseSnapshot,
events: [{ seq: 1, ts: 1, type: "model.usage", channel: null, durationMs: null }],
},
},
error: "snapshot.events[0].channel",
},
{
name: "optional-usage-before-context",
bundle: {
...baseBundle,
snapshot: {
...baseSnapshot,
events: [
{ seq: 1, ts: 1, type: "model.usage", usage: { input: null }, context: false },
],
},
},
error: "snapshot.events[0].usage.input",
},
{
name: "heap-statistics-before-required-memory",
bundle: {
...baseBundle,
evidence: {
memoryPressure: {
level: "critical",
reason: "rss_threshold",
heapStatistics: { totalHeapSizeBytes: null },
memory: null,
},
},
},
error: "evidence.memoryPressure.heapStatistics.totalHeapSizeBytes",
},
];
for (const testCase of cases) {
const file = path.join(tempDir, `${testCase.name}.json`);
fs.writeFileSync(file, `${JSON.stringify(testCase.bundle, null, 2)}\n`, "utf8");
const result = readDiagnosticStabilityBundleFileSync(file);
expect(result.status).toBe("failed");
expect(result.status === "failed" ? String(result.error) : "").toContain(testCase.error);
}
});
});