kimi-code / packages /minidb /test /bench-json.test.ts

Add files using upload-large-folder tool

4e23b01 verified about 2 months ago

8.5 kB

// Runs the real bench script in --quick mode and pins the machine-readable
// JSON report's shape: later phases diff before/after numbers, so the field
// names below are a stable contract.
import { test } from 'vitest';
import assert from 'node:assert/strict';
import { execFile } from 'node:child_process';
import { promisify } from 'node:util';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const pkgDir = fileURLToPath(new URL('..', import.meta.url));

test('bench --quick emits a JSON report with a stable schema', async () => {
  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-bench-json-'));
  try {
    const reportPath = path.join(dir, 'report.json');
    await promisify(execFile)(
      process.execPath,
      ['--import', 'tsx', path.join(pkgDir, 'bench', 'bench.ts'), '--quick', '--json', reportPath],
      { cwd: pkgDir, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
    );
    const report = JSON.parse(await fs.readFile(reportPath, 'utf8'));

    // Top-level envelope.
    assert.equal(report.schemaVersion, 1);
    assert.equal(report.tool, 'minidb/bench');
    assert.equal(typeof report.startedAt, 'string');
    assert.ok(!Number.isNaN(Date.parse(report.startedAt)), 'startedAt is an ISO timestamp');
    assert.equal(typeof report.node, 'string');
    assert.equal(typeof report.platform, 'string');
    assert.equal(typeof report.arch, 'string');
    assert.equal(typeof report.seed, 'number');
    assert.ok(Array.isArray(report.scenarios));
    assert.ok(report.scenarios.length >= 10, 'all scenario families ran');

    // Every scenario row carries the shared measurement fields.
    for (const s of report.scenarios) {
      assert.equal(typeof s.name, 'string');
      assert.ok(s.name.length > 0);
      assert.equal(typeof s.durationMs, 'number', `${s.name}.durationMs`);
      assert.ok(s.durationMs >= 0);
      if (s.ops !== undefined) assert.equal(typeof s.ops, 'number', `${s.name}.ops`);
      if (s.opsPerSec !== undefined) assert.equal(typeof s.opsPerSec, 'number', `${s.name}.opsPerSec`);
      for (const k of ['mean', 'p50', 'p95', 'p99', 'max']) {
        assert.equal(typeof s.eventLoopDelayMs[k], 'number', `${s.name}.eventLoopDelayMs.${k}`);
      }
      assert.ok(s.peakRssBytes > 0, `${s.name}.peakRssBytes`);
      assert.ok(s.peakHeapUsedBytes > 0, `${s.name}.peakHeapUsedBytes`);
      if (s.latencyMs !== undefined) {
        for (const k of ['p50', 'p95', 'p99', 'max']) {
          assert.equal(typeof s.latencyMs[k], 'number', `${s.name}.latencyMs.${k}`);
        }
      }
    }

    const byName = (re) => report.scenarios.find((s) => re.test(s.name));

    // Cold open rows expose the recovery/rebuild breakdown.
    const cold = byName(/cold open/);
    assert.ok(cold, 'a cold-open scenario exists');
    for (const k of ['keys', 'recoveryBytes', 'recoveryFrames', 'recoveryDurationMs', 'indexRebuildDurationMs', 'textRebuildDurationMs', 'walFsyncs']) {
      assert.equal(typeof cold.extra[k], 'number', `cold open extra.${k}`);
    }

    // The acceptance scenario: an idle everysec db performs zero fsyncs.
    const idle = byName(/idle everysec/);
    assert.ok(idle, 'the idle everysec scenario exists');
    assert.equal(idle.extra.walFsyncs, 0, 'idle everysec db: zero background fsyncs');
    assert.equal(idle.extra.walFsyncErrors, 0);

    // A write re-arms the background sync.
    const dirty = byName(/write then idle/);
    assert.ok(dirty, 'the dirty-window scenario exists');
    assert.ok(dirty.extra.walFsyncs >= 1, 'a write triggers a background fsync');

    // Search rows report per-query hit counts and medians.
    for (const re of [/search word/, /search ngram/]) {
      const search = byName(re);
      assert.ok(search, `${re} scenario exists`);
      assert.ok(Array.isArray(search.extra.queries));
      for (const q of search.extra.queries) {
        assert.equal(typeof q.q, 'string');
        assert.equal(typeof q.hits, 'number');
        assert.equal(typeof q.medianMs, 'number');
      }
    }

    // The compaction row exposes the phase breakdown.
    const compact = byName(/compact \d/);
    assert.ok(compact, 'a compaction scenario exists');
    for (const k of ['keys', 'compactionDurationMs', 'compactionSnapshotDurationMs', 'compactionRotationDurationMs', 'compactionPostingsDurationMs', 'snapshotBytesWritten']) {
      assert.equal(typeof compact.extra[k], 'number', `compact extra.${k}`);
    }
  } finally {
    await fs.rm(dir, { recursive: true, force: true });
  }
}, 300_000);

test('open-lifecycle bench --quick emits the phase-1 baseline report', async () => {
  const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-open-bench-json-'));
  try {
    const reportPath = path.join(dir, 'report.json');
    await promisify(execFile)(
      process.execPath,
      ['--import', 'tsx', path.join(pkgDir, 'bench', 'open-lifecycle.ts'), '--quick', '--json', reportPath],
      { cwd: pkgDir, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
    );
    const report = JSON.parse(await fs.readFile(reportPath, 'utf8'));

    // Top-level envelope.
    assert.equal(report.schemaVersion, 1);
    assert.equal(report.tool, 'minidb/bench/open-lifecycle');
    assert.ok(Array.isArray(report.scenarios));

    const byName = (re) => report.scenarios.find((s) => re.test(s.name));

    // Every row carries the event-loop-delay + input-latency baselines.
    for (const s of report.scenarios) {
      assert.equal(typeof s.durationMs, 'number', `${s.name}.durationMs`);
      for (const k of ['mean', 'p50', 'p95', 'p99', 'max']) {
        assert.equal(typeof s.eventLoopDelayMs[k], 'number', `${s.name}.eventLoopDelayMs.${k}`);
      }
      assert.ok(s.inputLatencyMs, `${s.name}.inputLatencyMs`);
      for (const k of ['count', 'p50', 'p95', 'p99', 'max']) {
        assert.equal(typeof s.inputLatencyMs[k], 'number', `${s.name}.inputLatencyMs.${k}`);
      }
    }

    // The four scenario families exist with their lifecycle phase breakdown.
    const PHASES = [
      'openMs',
      'generationCandidateLoadMs',
      'storeImageLoadMs',
      'nonTextImageLoadMs',
      'textImageLoadMs',
      'postingsIntegrityCheckMs',
      'walScanMs',
      'walApplyMs',
      'fullRecoveryMs',
      'textRebuildMs',
    ];
    const small = byName(/small corpus, healthy generation/);
    const wal = byName(/large WAL delta/);
    const large = byName(/large full-text generation/);
    const corrupt = byName(/corrupt generation \(full-recovery fallback\)/);
    const deferred = byName(/deferred text rebuild \(degraded -> ready\)/);
    for (const [name, s] of Object.entries({ small, wal, large, corrupt, deferred })) {
      assert.ok(s, `${name} scenario exists`);
      for (const k of PHASES) assert.equal(typeof s.extra.phases[k], 'number', `${name}.extra.phases.${k}`);
    }

    // Healthy generation opens: no full recovery, no corpus tokenization.
    for (const [name, s] of Object.entries({ small, wal, large })) {
      assert.equal(s.extra.state, 'ready', name);
      assert.deepEqual(s.extra.path, ['no-generation', 'generation-load', 'wal-catch-up', 'ready'], name);
      assert.equal(s.extra.phases.fullRecoveryMs, 0, name);
      assert.equal(s.extra.phases.textRebuildMs, 0, name);
      assert.equal(s.extra.generationLoads, 1, name);
      assert.equal(s.extra.indexRebuildDecoded, 0, name);
    }
    // The WAL-delta open really replayed the delta (and close did NOT
    // republish it into a fresh generation — the scenario would be void).
    assert.ok(wal.extra.walDeltaAppliedOps > 0, 'wal delta replayed');
    assert.equal(wal.extra.closePublishedGeneration, false, 'close left the generation untouched');
    assert.ok(wal.extra.phases.walApplyMs > 0, 'wal apply measured');

    // The corrupt generation fell back to a full recovery and returned
    // degraded; the background deferred build then brought it to ready.
    assert.equal(corrupt.extra.state, 'degraded');
    assert.deepEqual(corrupt.extra.path, ['no-generation', 'generation-load', 'full-rebuild', 'degraded']);
    assert.ok(corrupt.extra.phases.fullRecoveryMs > 0, 'full recovery measured');
    assert.equal(corrupt.extra.generationLoadFallbacks, 1);
    assert.equal(deferred.extra.state, 'ready');
    assert.ok(deferred.extra.textDeferredBuilds >= 1, 'deferred build committed');
  } finally {
    await fs.rm(dir, { recursive: true, force: true });
  }
}, 300_000);

	// Runs the real bench script in --quick mode and pins the machine-readable
	// JSON report's shape: later phases diff before/after numbers, so the field
	// names below are a stable contract.
	import { test } from 'vitest';
	import assert from 'node:assert/strict';
	import { execFile } from 'node:child_process';
	import { promisify } from 'node:util';
	import fs from 'node:fs/promises';
	import os from 'node:os';
	import path from 'node:path';
	import { fileURLToPath } from 'node:url';

	const pkgDir = fileURLToPath(new URL('..', import.meta.url));

	test('bench --quick emits a JSON report with a stable schema', async () => {
	const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-bench-json-'));
	try {
	const reportPath = path.join(dir, 'report.json');
	await promisify(execFile)(
	process.execPath,
	['--import', 'tsx', path.join(pkgDir, 'bench', 'bench.ts'), '--quick', '--json', reportPath],
	{ cwd: pkgDir, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
	);
	const report = JSON.parse(await fs.readFile(reportPath, 'utf8'));

	// Top-level envelope.
	assert.equal(report.schemaVersion, 1);
	assert.equal(report.tool, 'minidb/bench');
	assert.equal(typeof report.startedAt, 'string');
	assert.ok(!Number.isNaN(Date.parse(report.startedAt)), 'startedAt is an ISO timestamp');
	assert.equal(typeof report.node, 'string');
	assert.equal(typeof report.platform, 'string');
	assert.equal(typeof report.arch, 'string');
	assert.equal(typeof report.seed, 'number');
	assert.ok(Array.isArray(report.scenarios));
	assert.ok(report.scenarios.length >= 10, 'all scenario families ran');

	// Every scenario row carries the shared measurement fields.
	for (const s of report.scenarios) {
	assert.equal(typeof s.name, 'string');
	assert.ok(s.name.length > 0);
	assert.equal(typeof s.durationMs, 'number', `${s.name}.durationMs`);
	assert.ok(s.durationMs >= 0);
	if (s.ops !== undefined) assert.equal(typeof s.ops, 'number', `${s.name}.ops`);
	if (s.opsPerSec !== undefined) assert.equal(typeof s.opsPerSec, 'number', `${s.name}.opsPerSec`);
	for (const k of ['mean', 'p50', 'p95', 'p99', 'max']) {
	assert.equal(typeof s.eventLoopDelayMs[k], 'number', `${s.name}.eventLoopDelayMs.${k}`);
	}
	assert.ok(s.peakRssBytes > 0, `${s.name}.peakRssBytes`);
	assert.ok(s.peakHeapUsedBytes > 0, `${s.name}.peakHeapUsedBytes`);
	if (s.latencyMs !== undefined) {
	for (const k of ['p50', 'p95', 'p99', 'max']) {
	assert.equal(typeof s.latencyMs[k], 'number', `${s.name}.latencyMs.${k}`);
	}
	}
	}

	const byName = (re) => report.scenarios.find((s) => re.test(s.name));

	// Cold open rows expose the recovery/rebuild breakdown.
	const cold = byName(/cold open/);
	assert.ok(cold, 'a cold-open scenario exists');
	for (const k of ['keys', 'recoveryBytes', 'recoveryFrames', 'recoveryDurationMs', 'indexRebuildDurationMs', 'textRebuildDurationMs', 'walFsyncs']) {
	assert.equal(typeof cold.extra[k], 'number', `cold open extra.${k}`);
	}

	// The acceptance scenario: an idle everysec db performs zero fsyncs.
	const idle = byName(/idle everysec/);
	assert.ok(idle, 'the idle everysec scenario exists');
	assert.equal(idle.extra.walFsyncs, 0, 'idle everysec db: zero background fsyncs');
	assert.equal(idle.extra.walFsyncErrors, 0);

	// A write re-arms the background sync.
	const dirty = byName(/write then idle/);
	assert.ok(dirty, 'the dirty-window scenario exists');
	assert.ok(dirty.extra.walFsyncs >= 1, 'a write triggers a background fsync');

	// Search rows report per-query hit counts and medians.
	for (const re of [/search word/, /search ngram/]) {
	const search = byName(re);
	assert.ok(search, `${re} scenario exists`);
	assert.ok(Array.isArray(search.extra.queries));
	for (const q of search.extra.queries) {
	assert.equal(typeof q.q, 'string');
	assert.equal(typeof q.hits, 'number');
	assert.equal(typeof q.medianMs, 'number');
	}
	}

	// The compaction row exposes the phase breakdown.
	const compact = byName(/compact \d/);
	assert.ok(compact, 'a compaction scenario exists');
	for (const k of ['keys', 'compactionDurationMs', 'compactionSnapshotDurationMs', 'compactionRotationDurationMs', 'compactionPostingsDurationMs', 'snapshotBytesWritten']) {
	assert.equal(typeof compact.extra[k], 'number', `compact extra.${k}`);
	}
	} finally {
	await fs.rm(dir, { recursive: true, force: true });
	}
	}, 300_000);

	test('open-lifecycle bench --quick emits the phase-1 baseline report', async () => {
	const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'minidb-open-bench-json-'));
	try {
	const reportPath = path.join(dir, 'report.json');
	await promisify(execFile)(
	process.execPath,
	['--import', 'tsx', path.join(pkgDir, 'bench', 'open-lifecycle.ts'), '--quick', '--json', reportPath],
	{ cwd: pkgDir, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
	);
	const report = JSON.parse(await fs.readFile(reportPath, 'utf8'));

	// Top-level envelope.
	assert.equal(report.schemaVersion, 1);
	assert.equal(report.tool, 'minidb/bench/open-lifecycle');
	assert.ok(Array.isArray(report.scenarios));

	const byName = (re) => report.scenarios.find((s) => re.test(s.name));

	// Every row carries the event-loop-delay + input-latency baselines.
	for (const s of report.scenarios) {
	assert.equal(typeof s.durationMs, 'number', `${s.name}.durationMs`);
	for (const k of ['mean', 'p50', 'p95', 'p99', 'max']) {
	assert.equal(typeof s.eventLoopDelayMs[k], 'number', `${s.name}.eventLoopDelayMs.${k}`);
	}
	assert.ok(s.inputLatencyMs, `${s.name}.inputLatencyMs`);
	for (const k of ['count', 'p50', 'p95', 'p99', 'max']) {
	assert.equal(typeof s.inputLatencyMs[k], 'number', `${s.name}.inputLatencyMs.${k}`);
	}
	}

	// The four scenario families exist with their lifecycle phase breakdown.
	const PHASES = [
	'openMs',
	'generationCandidateLoadMs',
	'storeImageLoadMs',
	'nonTextImageLoadMs',
	'textImageLoadMs',
	'postingsIntegrityCheckMs',
	'walScanMs',
	'walApplyMs',
	'fullRecoveryMs',
	'textRebuildMs',
	];
	const small = byName(/small corpus, healthy generation/);
	const wal = byName(/large WAL delta/);
	const large = byName(/large full-text generation/);
	const corrupt = byName(/corrupt generation \(full-recovery fallback\)/);
	const deferred = byName(/deferred text rebuild \(degraded -> ready\)/);
	for (const [name, s] of Object.entries({ small, wal, large, corrupt, deferred })) {
	assert.ok(s, `${name} scenario exists`);
	for (const k of PHASES) assert.equal(typeof s.extra.phases[k], 'number', `${name}.extra.phases.${k}`);
	}

	// Healthy generation opens: no full recovery, no corpus tokenization.
	for (const [name, s] of Object.entries({ small, wal, large })) {
	assert.equal(s.extra.state, 'ready', name);
	assert.deepEqual(s.extra.path, ['no-generation', 'generation-load', 'wal-catch-up', 'ready'], name);
	assert.equal(s.extra.phases.fullRecoveryMs, 0, name);
	assert.equal(s.extra.phases.textRebuildMs, 0, name);
	assert.equal(s.extra.generationLoads, 1, name);
	assert.equal(s.extra.indexRebuildDecoded, 0, name);
	}
	// The WAL-delta open really replayed the delta (and close did NOT
	// republish it into a fresh generation — the scenario would be void).
	assert.ok(wal.extra.walDeltaAppliedOps > 0, 'wal delta replayed');
	assert.equal(wal.extra.closePublishedGeneration, false, 'close left the generation untouched');
	assert.ok(wal.extra.phases.walApplyMs > 0, 'wal apply measured');

	// The corrupt generation fell back to a full recovery and returned
	// degraded; the background deferred build then brought it to ready.
	assert.equal(corrupt.extra.state, 'degraded');
	assert.deepEqual(corrupt.extra.path, ['no-generation', 'generation-load', 'full-rebuild', 'degraded']);
	assert.ok(corrupt.extra.phases.fullRecoveryMs > 0, 'full recovery measured');
	assert.equal(corrupt.extra.generationLoadFallbacks, 1);
	assert.equal(deferred.extra.state, 'ready');
	assert.ok(deferred.extra.textDeferredBuilds >= 1, 'deferred build committed');
	} finally {
	await fs.rm(dir, { recursive: true, force: true });
	}
	}, 300_000);