// test/db.test.js import { test } from 'vitest'; import assert from 'node:assert/strict'; import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { MiniDb } from '../src/index.js'; import { encodeFrame, TYPE_SET } from '../src/codec.js'; import { barrier, waitFor } from './helpers.js'; const B = (s) => Buffer.from(s); async function tmpDir() { return fs.mkdtemp(path.join(os.tmpdir(), 'minidb-db-')); } test('set/get persists across reopen (string codec)', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'everysec' }); await db.set('a', '1'); await db.set('b', '2'); assert.equal(db.get('a'), '1'); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(db.get('a'), '1'); assert.equal(db.get('b'), '2'); assert.equal(db.size, 2); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('del persists across reopen', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); await db.set('x', '1'); await db.set('y', '2'); await db.del('x'); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(db.get('x'), undefined); assert.equal(db.get('y'), '2'); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('json codec round-trips values', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'json' }); await db.set('obj', { a: 1, b: [2, 3] }); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'json' }); assert.deepEqual(db.get('obj'), { a: 1, b: [2, 3] }); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('ttl / expire', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', activeExpireIntervalMs: 0 }); await db.set('t', 'v', { ttl: 50 }); assert.ok(db.ttl('t') > 0 && db.ttl('t') <= 50); assert.equal(db.ttl('nope'), -2); await db.expire('t', 1000); assert.ok(db.ttl('t') > 50); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('recovery truncates a torn WAL tail and keeps valid data', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); for (let i = 0; i < 100; i++) await db.set(`k${i}`, `v${i}`); await db.close(); // Append a half-written frame to simulate a crash mid-write. const walPath = path.join(dir, 'db.wal'); const partial = encodeFrame({ type: TYPE_SET, key: B('torn'), value: B('x'.repeat(200)) }).subarray(0, 11); await fs.appendFile(walPath, partial); db = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(db.recoveryInfo.truncatedWal, true); assert.equal(db.size, 100); assert.equal(db.get('k0'), 'v0'); assert.equal(db.get('k99'), 'v99'); await db.close(); // The torn tail is gone for good. const reopened = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(reopened.recoveryInfo.truncatedWal, false); assert.equal(reopened.size, 100); await reopened.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('maxMemory reject policy blocks writes over budget', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'reject' }); await db.set('a', '1234567890'); await db.set('b', '1234567890'); await db.set('c', '1234567890'); await db.set('d', '1234567890'); await assert.rejects(() => db.set('e', '1234567890'), /maxMemory exceeded/); assert.equal(db.get('e'), undefined); assert.ok(db.stats.maxMemoryRejections >= 1); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('maxMemory evict-lru evicts old keys to make room', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'evict-lru' }); await db.set('a', '1234567890'); await db.set('b', '1234567890'); await db.set('c', '1234567890'); await db.set('d', '1234567890'); assert.equal(db.get('a'), '1234567890'); // touch a; b becomes the LRU victim await db.set('e', '1234567890'); assert.equal(db.get('a'), '1234567890'); assert.equal(db.get('b'), undefined); assert.equal(db.get('e'), '1234567890'); assert.ok(db.stats.evictions >= 1); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('backup + restore preserves data, indexes, and text search', async () => { const dir = await tmpDir(); const backupDir = await tmpDir(); const restoreDir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'json' }); await db.createIndex('byCity', { field: 'city' }); await db.createTextIndex('body', { fields: ['body'] }); await db.set('a', { city: 'Paris', body: 'hello world' }); await db.set('b', { city: 'London', body: 'hello London' }); await db.backup(backupDir); await db.close(); const restored = await MiniDb.restore(backupDir, restoreDir, { valueCodec: 'json' }); assert.deepEqual(restored.get('a'), { city: 'Paris', body: 'hello world' }); assert.deepEqual(restored.findEq('byCity', 'London').map((r) => r.key), ['b']); assert.deepEqual(restored.search('body', 'hello').map((r) => r.key).sort(), ['a', 'b']); await restored.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); await fs.rm(backupDir, { recursive: true, force: true }); await fs.rm(restoreDir, { recursive: true, force: true }); } }); test('backup fences writes at a linearization point: every pre-fence ack is in, concurrent writes reject (review #22)', async () => { const dir = await tmpDir(); const parent = await tmpDir(); const backupDir = path.join(parent, 'backup-dest'); // intentionally absent const restoreDir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', autoCompact: false }); await db.set('seed1', 'a'); await db.set('seed2', 'b'); // Park the WAL writev of the NEXT write so a set is provably in flight // when the backup starts: the backup's drain must wait for it, and its // ack then lands INSIDE the backup (it entered before the fence). const fh = (db as unknown as { wal: { fh: { writev: (...a: unknown[]) => Promise } } }).wal.fh; const gate = barrier(fh, 'writev', 1); const inFlight = db.set('inflight', 'c'); await gate.entered; const backingUp = db.backup(backupDir, { compact: false }); let backupDone = false; void backingUp.then(() => { backupDone = true; }); // Writes submitted behind the fence reject with a clear, retryable error. await assert.rejects(db.set('late', 'd'), (e: unknown) => (e as { code?: string }).code === 'BACKUP_IN_PROGRESS'); await assert.rejects(db.batch([{ op: 'set', key: 'late2', value: 'd' }]), /backup is in progress/i); // The backup cannot complete while the fenced-in write is still parked. for (let i = 0; i < 5; i++) await new Promise((r) => setImmediate(r)); assert.equal(backupDone, false, 'backup waits for the in-flight write (its linearization point)'); gate.release(); gate.restore(); await inFlight; await backingUp; assert.equal(backupDone, true); // After the backup the gate is lifted: writes work again. await db.set('after', 'e'); assert.equal(db.get('after'), 'e'); await db.close(); // The manifest is the commit marker and lists the copied files. const manifest = JSON.parse(await fs.readFile(path.join(backupDir, 'backup.manifest.json'), 'utf8')) as { files: string[] }; assert.ok(manifest.files.includes('db.wal'), 'manifest lists the WAL'); // Restore: every write ack'd before the fence is in; the rejected ones are not. const restored = await MiniDb.restore(backupDir, restoreDir, { valueCodec: 'string' }); assert.equal(restored.get('seed1'), 'a'); assert.equal(restored.get('seed2'), 'b'); assert.equal(restored.get('inflight'), 'c', 'the write drained before the fence is included'); assert.equal(restored.get('late'), undefined); assert.equal(restored.get('late2'), undefined); assert.equal(restored.get('after'), undefined, 'a post-backup write is not included'); await restored.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); await fs.rm(parent, { recursive: true, force: true }); await fs.rm(restoreDir, { recursive: true, force: true }); } }); test('backup copy failure leaves no partial backup behind and reopens the write gate (review #22)', async () => { const dir = await tmpDir(); const parent = await tmpDir(); const backupDir = path.join(parent, 'backup-dest'); try { const db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no', autoCompact: false }); await db.set('k', 'v'); // Fail the copy phase: every fs.copyFile rejects. const boom = new Error('injected copy failure'); const origCopy = fs.copyFile; (fs as unknown as { copyFile: unknown }).copyFile = () => Promise.reject(boom); try { await assert.rejects(db.backup(backupDir, { compact: false }), /injected copy failure/); } finally { (fs as unknown as { copyFile: unknown }).copyFile = origCopy; } // No half backup: the destination was never created, and no temp/aside // dir is stranded next to it. assert.equal(await fs.stat(backupDir).then(() => true, () => false), false, 'destination untouched'); const leftovers = (await fs.readdir(parent)).filter((n) => n.includes('.backup-')); assert.deepEqual(leftovers, [], `stranded temp dirs: ${leftovers.join(', ')}`); // The write gate was released: the db keeps working. await db.set('after', 'w'); assert.equal(db.get('after'), 'w'); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); await fs.rm(parent, { recursive: true, force: true }); } }); test('readOnly open of a non-existent directory fails with ENOENT and creates nothing (review #26)', async () => { const parent = await tmpDir(); const dir = path.join(parent, 'missing'); try { await assert.rejects( MiniDb.open({ dir, valueCodec: 'string', readOnly: true }), (e: unknown) => (e as NodeJS.ErrnoException).code === 'ENOENT', ); assert.equal(await fs.stat(dir).then(() => true, () => false), false, 'no directory was created'); } finally { await fs.rm(parent, { recursive: true, force: true }); } }); test('valueMode disk stores value pointers and reads from WAL', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); const big = 'x'.repeat(1000); await db.set('a', big); const rec = db.store.map.get('a'); assert.equal(rec?.ref.kind, 'disk'); assert.equal(rec?.ref.kind === 'disk' && rec.ref.loc.file, 'wal'); assert.equal(db.get('a'), big); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); const rec2 = db.store.map.get('a'); assert.equal(rec2?.ref.kind, 'disk'); assert.equal(db.get('a'), big); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode disk batch stores value pointers and survives reopen', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); await db.batch([ { op: 'set', key: 'a', value: '1'.repeat(100) }, { op: 'set', key: 'b', value: '2'.repeat(100) }, ]); assert.equal(db.store.map.get('a')?.ref.kind, 'disk'); assert.equal(db.store.map.get('b')?.ref.kind, 'disk'); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); assert.equal(db.get('a'), '1'.repeat(100)); assert.equal(db.get('b'), '2'.repeat(100)); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode disk compaction remaps pointers to the snapshot', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', fsyncPolicy: 'no' }); await db.set('a', '1'.repeat(1000)); await db.set('b', '2'.repeat(1000)); await db.compact(); const rec = db.store.map.get('a'); assert.equal(rec?.ref.kind, 'disk'); assert.equal(rec?.ref.kind === 'disk' && rec.ref.loc.file, 'snapshot'); assert.equal(db.get('a'), '1'.repeat(1000)); assert.equal(db.get('b'), '2'.repeat(1000)); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk' }); assert.equal(db.get('a'), '1'.repeat(1000)); assert.equal(db.get('b'), '2'.repeat(1000)); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode disk maxMemory excludes value bulk', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'disk', maxMemoryBytes: 200, maxMemoryPolicy: 'reject', fsyncPolicy: 'no', }); await db.set('a', 'x'.repeat(1000)); await db.set('b', 'y'.repeat(1000)); await db.set('c', 'z'.repeat(1000)); assert.equal(db.get('a'), 'x'.repeat(1000)); assert.ok(db.store.bytes < 200); await assert.rejects(() => db.set('d', 'w'.repeat(1000)), /maxMemory exceeded/); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode auto selects disk when persisted files exceed maxMemoryBytes', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); await db.set('a', 'x'.repeat(1000)); await db.close(); const walSize = (await fs.stat(path.join(dir, 'db.wal'))).size; db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'auto', maxMemoryBytes: Math.max(1, walSize - 1), fsyncPolicy: 'no', }); assert.equal(db.valueMode, 'disk'); assert.equal(db.store.map.get('a')?.ref.kind, 'disk'); assert.equal(db.get('a'), 'x'.repeat(1000)); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode auto selects memory when persisted files fit maxMemoryBytes', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); await db.set('a', 'x'.repeat(1000)); await db.close(); const walSize = (await fs.stat(path.join(dir, 'db.wal'))).size; db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'auto', maxMemoryBytes: walSize + 1000, fsyncPolicy: 'no', }); assert.equal(db.valueMode, 'memory'); assert.equal(db.store.map.get('a')?.ref.kind, 'memory'); assert.equal(db.get('a'), 'x'.repeat(1000)); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('valueMode auto without maxMemoryBytes defaults to memory', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); await db.set('a', 'x'.repeat(1000)); await db.close(); db = await MiniDb.open({ dir, valueCodec: 'string', valueMode: 'auto', fsyncPolicy: 'no' }); assert.equal(db.valueMode, 'memory'); assert.equal(db.store.map.get('a')?.ref.kind, 'memory'); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('openOrRebuild preserves data when only a sidecar definition file is corrupt', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); for (let i = 0; i < 100; i++) await db.set(`k${i}`, { n: i }); await db.createIndex('byN', { field: 'n' }); await db.close(); await fs.writeFile(path.join(dir, 'db.indexes.json'), 'corrupt{{{'); // plain open still throws on the corrupt sidecar... await assert.rejects(MiniDb.open({ dir, valueCodec: 'json' }), SyntaxError); // ...but openOrRebuild drops the derived sidecars instead of wiping the data db = await MiniDb.openOrRebuild>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); assert.equal(db.size, 100); assert.deepEqual(db.get('k42'), { n: 42 }); assert.deepEqual(db.listIndexes(), []); // definitions are lost, not the data; recreate as needed await db.createIndex('byN2', { field: 'n' }); assert.equal(db.findEq('byN2', 42).length, 1); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('open removes stale compaction temp files', async () => { const dir = await tmpDir(); try { let db = await MiniDb.open({ dir, valueCodec: 'string', fsyncPolicy: 'no' }); await db.set('a', '1'); await db.close(); await fs.writeFile(path.join(dir, 'db.snapshot.tmp'), 'stale'); await fs.writeFile(path.join(dir, 'db.wal.tmp'), 'stale'); db = await MiniDb.open({ dir, valueCodec: 'string' }); assert.equal(db.get('a'), '1'); await assert.rejects(fs.stat(path.join(dir, 'db.snapshot.tmp')), /ENOENT/); await assert.rejects(fs.stat(path.join(dir, 'db.wal.tmp')), /ENOENT/); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('query with skip/limit returns the same rows as slicing the unbounded result', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open>({ dir, valueCodec: 'json', fsyncPolicy: 'no' }); for (let i = 0; i < 200; i++) await db.set(`k${String(i).padStart(3, '0')}`, { n: i, grp: i % 4 }); await db.createIndex('byGrp', { field: 'grp' }); const full = db.query({ key: { prefix: 'k1' } }); assert.deepEqual(db.query({ key: { prefix: 'k1' }, limit: 10 }), full.slice(0, 10)); assert.deepEqual(db.query({ key: { prefix: 'k1' }, skip: 5, limit: 3 }), full.slice(5, 8)); const eq = db.query({ filter: { grp: 2 } }); // indexed equality path assert.deepEqual(db.query({ filter: { grp: 2 }, limit: 7 }), eq.slice(0, 7)); const scan = db.query({ filter: { n: { $gte: 100 } } }); // unindexed full scan path assert.deepEqual(db.query({ filter: { n: { $gte: 100 } }, skip: 10, limit: 5 }), scan.slice(10, 15)); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); test('maxMemory evict-lru evicts in least-recently-used order across many victims', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open({ dir, valueCodec: 'string', maxMemoryBytes: 50, maxMemoryPolicy: 'evict-lru' }); await db.set('a', '1234567890'); // ~11B per record, budget fits 4 await db.set('b', '1234567890'); await db.set('c', '1234567890'); await db.set('d', '1234567890'); assert.equal(db.get('a'), '1234567890'); // touch a: b becomes LRU await db.set('e', '1234567890'); // exceeds budget -> evicts b await db.set('f', '1234567890'); // exceeds budget -> evicts c assert.equal(db.get('a'), '1234567890'); assert.equal(db.get('b'), undefined); assert.equal(db.get('c'), undefined); assert.equal(db.get('d'), '1234567890'); assert.equal(db.get('e'), '1234567890'); assert.equal(db.get('f'), '1234567890'); await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); // ---- stage 6: async read APIs ------------------------------------------------- for (const valueMode of ['memory', 'disk'] as const) { test(`async reads match sync reads (valueMode: ${valueMode})`, async () => { const dir = await tmpDir(); try { const db = await MiniDb.open>({ dir, valueCodec: 'json', valueMode, autoCompact: false }); await db.createTextIndex('ft', { fields: ['text'] }); await db.createIndex('byKind', { field: 'kind' }); for (let i = 0; i < 300; i++) { await db.set(`k${i}`, { kind: `t${i % 5}`, score: i, text: `hello world doc ${i} 异步 读取 ${i % 7}` }, { dt: { ts: 1700000000000 + i } }); } await db.del('k0'); await db.set('k1', { kind: 't1', score: 999, text: 'replaced 替换' }); // getAsync vs get assert.deepEqual(await db.getAsync('k1'), db.get('k1')); assert.deepEqual(await db.getAsync('k42'), db.get('k42')); assert.equal(await db.getAsync('k0'), undefined); assert.equal(await db.getAsync('missing'), undefined); // searchBoundedAsync / searchAsync vs search for (const q of ['hello', '异步', 'hello world', 'replaced']) { const syncRes = db.searchBounded('ft', q, { limit: 20 }); const asyncRes = await db.searchBoundedAsync('ft', q, { limit: 20 }); assert.deepEqual(asyncRes, syncRes, `searchBoundedAsync(${q})`); assert.deepEqual(await db.searchAsync('ft', q, { limit: 20 }), db.search('ft', q, { limit: 20 })); } // queryAsync vs query across the planner's shapes const shapes = [ { filter: { kind: 't3' } }, { dt: { ts: { gte: 1700000000050 } }, limit: 10 }, { dt: { ts: { gte: 1700000000050 } }, sort: { ts: -1 as const }, limit: 5 }, { text: { index: 'ft', q: 'hello' }, limit: 10 }, { key: { prefix: 'k1' }, limit: 5 }, { sort: { score: -1 as const }, limit: 7 }, { filter: { kind: 't2' }, sort: { score: 1 as const }, skip: 3, limit: 6 }, ]; for (const q of shapes) { assert.deepEqual(await db.queryAsync(q), db.query(q), `queryAsync(${JSON.stringify(q)})`); } await db.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } }); } test('async reads stay correct across a compaction rotation (disk mode)', async () => { const dir = await tmpDir(); try { const db = await MiniDb.open>({ dir, valueCodec: 'json', valueMode: 'disk', compactThresholdBytes: 4096 }); for (let i = 0; i < 400; i++) { await db.set(`k${i}`, { kind: `t${i % 5}`, score: i, text: `rotation doc ${i}` }); } await waitFor(() => db.stats.compactions >= 1, 'auto compaction'); for (let i = 400; i < 500; i++) { await db.set(`k${i}`, { kind: 't6', score: i, text: `post rotation doc ${i}` }); } for (let i = 0; i < 500; i += 37) { assert.deepEqual(await db.getAsync(`k${i}`), db.get(`k${i}`)); } await db.close(); // read-only reopen: async reads identical to sync reads against the rotated files. const ro = await MiniDb.open>({ dir, valueCodec: 'json', valueMode: 'disk', readOnly: true }); for (let i = 0; i < 500; i += 41) { assert.deepEqual(await ro.getAsync(`k${i}`), ro.get(`k${i}`)); } await ro.close(); } finally { await fs.rm(dir, { recursive: true, force: true }); } });