Spaces:

automindy
/

fokemal

Sleeping

fokemal / server /src /__tests__ /services /cache.test.ts
Sync to upstream v0.13.7, keep HF Space adaptations
dcedb25 21 days ago
32.4 kB
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
import { initDb, setSetting, getDb } from '../../db/index.js';
import {
  computeCacheKey,
  getCachedResponse,
  storeCachedResponse,
  getCachedStreamResponse,
  storeCachedStreamResponse,
  STREAM_CACHE_MAX_BYTES,
  getCacheStats,
  clearCache,
  loadCacheFromDb,
  __flushPersistenceForTests,
  __resetMemoryForTests,
  isCacheableTemperature,
  parseCacheDirective,
  cacheActive,
  isCacheEnabled,
  CACHE_ENABLED_SETTING,
} from '../../services/cache.js';
import type { ChatMessage } from '@freellmapi/shared/types.js';

const CACHE_ENV = [
  'RESPONSE_CACHE',
  'RESPONSE_CACHE_PERSIST',
  'RESPONSE_CACHE_TTL_SECONDS',
  'RESPONSE_CACHE_MAX_TEMPERATURE',
  'RESPONSE_CACHE_MAX_ENTRIES',
] as const;

function msg(role: ChatMessage['role'], content: string): ChatMessage {
  return { role, content } as ChatMessage;
}

const sampleBody = (text: string) => ({
  id: 'chatcmpl-test',
  object: 'chat.completion',
  choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }],
  usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
});

const store = (key: string, text: string, now?: number) =>
  storeCachedResponse(key, {
    body: sampleBody(text),
    platform: 'groq',
    modelId: 'llama-3.3-70b',
    keyId: 7,
    promptTokens: 10,
    completionTokens: 5,
  }, now);

describe('response cache', () => {
  const saved: Record<string, string | undefined> = {};

  beforeEach(() => {
    process.env.ENCRYPTION_KEY = '0'.repeat(64);
    for (const k of CACHE_ENV) saved[k] = process.env[k];
    // Fresh in-memory DB per test so the settings-backed toggle is isolated.
    initDb(':memory:');
    // The cache is module-level in-memory state: flush it between tests.
    clearCache();
  });

  afterEach(() => {
    clearCache();
    for (const k of CACHE_ENV) {
      if (saved[k] === undefined) delete process.env[k];
      else process.env[k] = saved[k]!;
    }
  });

  describe('computeCacheKey', () => {
    it('is stable regardless of object key order', () => {
      const messages = [msg('user', 'hello')];
      const a = computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 1 });
      const b = computeCacheKey({ messages, top_p: 1, temperature: 0.2, model: 'auto' });
      expect(a).toBe(b);
    });

    it('treats omitted model and "auto" as the same bucket', () => {
      const messages = [msg('user', 'hello')];
      expect(computeCacheKey({ model: undefined, messages }))
        .toBe(computeCacheKey({ model: 'auto', messages }));
    });

    it('changes when the prompt changes', () => {
      expect(computeCacheKey({ model: 'auto', messages: [msg('user', 'a')] }))
        .not.toBe(computeCacheKey({ model: 'auto', messages: [msg('user', 'b')] }));
    });

    it('changes when sampling params change', () => {
      const messages = [msg('user', 'hello')];
      expect(computeCacheKey({ model: 'auto', messages, temperature: 0 }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, temperature: 0.9 }));
    });

    // Explicit-but-default-valued sampling params must NOT change the key: a
    // client that always serializes top_p:1 / n:1 / presence_penalty:0 /
    // frequency_penalty:0 must hit the same entry as one that omits them.
    it('ignores explicit default-valued sampling params', () => {
      const messages = [msg('user', 'hello')];
      const minimal = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });
      const verbose = computeCacheKey({
        model: 'auto',
        messages,
        temperature: 0.2,
        top_p: 1,
        presence_penalty: 0,
        frequency_penalty: 0,
        n: 1,
      });
      expect(verbose).toBe(minimal);
    });

    it('still keys non-default sampling param values', () => {
      const messages = [msg('user', 'hello')];
      const baseline = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });
      expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 0.9 }))
        .not.toBe(baseline);
      expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, presence_penalty: 0.5 }))
        .not.toBe(baseline);
      expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, frequency_penalty: 0.5 }))
        .not.toBe(baseline);
      expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, n: 2 }))
        .not.toBe(baseline);
    });

    it('distinguishes a pinned model from auto', () => {
      const messages = [msg('user', 'hello')];
      expect(computeCacheKey({ model: 'gpt-oss-120b', messages }))
        .not.toBe(computeCacheKey({ model: 'auto', messages }));
    });

    // Tools policy: a tool-bearing request is cacheable, but its tool set is
    // part of the key, so a different tool set is a MISS and never serves
    // another request's cached answer.
    it('changes when the tool set changes (no cross-tool contamination)', () => {
      const messages = [msg('user', 'call a tool')];
      const toolsA = [{ type: 'function', function: { name: 'Read' } }];
      const toolsB = [{ type: 'function', function: { name: 'Write' } }];
      expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, tools: toolsB }));
      // Same tools + same prompt collapse to the same key.
      expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
        .toBe(computeCacheKey({ model: 'auto', messages, tools: toolsA }));
      // A tools request differs from the tool-free version of the same prompt.
      expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
        .not.toBe(computeCacheKey({ model: 'auto', messages }));
    });

    // Collision guard for the remaining sampling/format knobs: requests that
    // differ ONLY in one of these must never share a key, or one could be
    // served the other's cached answer (worst case: a response_format
    // json_object request replayed a cached plain-text reply).
    it('changes when response_format changes (json_object vs none)', () => {
      const messages = [msg('user', 'give me data')];
      expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))
        .not.toBe(computeCacheKey({ model: 'auto', messages }));
      expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, response_format: { type: 'text' } }));
    });

    it('changes when stop, seed, or n change', () => {
      const messages = [msg('user', 'hello')];
      const base = computeCacheKey({ model: 'auto', messages });
      expect(computeCacheKey({ model: 'auto', messages, stop: ['\n'] })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, stop: 'END' }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, stop: 'STOP' }));
      expect(computeCacheKey({ model: 'auto', messages, seed: 42 })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, seed: 42 }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, seed: 43 }));
      expect(computeCacheKey({ model: 'auto', messages, n: 2 })).not.toBe(base);
    });

    it('changes when penalties, logit_bias, or logprobs knobs change', () => {
      const messages = [msg('user', 'hello')];
      const base = computeCacheKey({ model: 'auto', messages });
      expect(computeCacheKey({ model: 'auto', messages, presence_penalty: 0.5 })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, frequency_penalty: 0.5 })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, logit_bias: { '50256': -100 } })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, logprobs: true })).not.toBe(base);
      expect(computeCacheKey({ model: 'auto', messages, logprobs: true, top_logprobs: 5 }))
        .not.toBe(computeCacheKey({ model: 'auto', messages, logprobs: true }));
    });

    it('identical requests with the new knobs present collapse to the same key', () => {
      const messages = [msg('user', 'give me data')];
      const full = () => computeCacheKey({
        model: 'auto', messages, temperature: 0.1,
        response_format: { type: 'json_object' },
        stop: ['END', 'STOP'], n: 1, seed: 42,
        presence_penalty: 0.25, frequency_penalty: 0.5,
        logit_bias: { '50256': -100 }, logprobs: true, top_logprobs: 3,
      });
      expect(full()).toBe(full());
      // Key-order independence holds for the new fields too.
      expect(computeCacheKey({ seed: 42, stop: 'END', model: 'auto', messages } as any))
        .toBe(computeCacheKey({ model: 'auto', messages, stop: 'END', seed: 42 }));
      // And absent knobs still hash like before (undefined dropped, not null-ed).
      expect(computeCacheKey({ model: 'auto', messages, seed: undefined }))
        .toBe(computeCacheKey({ model: 'auto', messages }));
    });
  });

  describe('store / get round-trip', () => {
    it('returns null on a miss', () => {
      expect(getCachedResponse('does-not-exist')).toBeNull();
    });

    it('returns the stored body on a hit', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      store(key, 'cached answer');
      const hit = getCachedResponse(key);
      expect(hit).not.toBeNull();
      expect((hit!.body as any).choices[0].message.content).toBe('cached answer');
      expect(hit!.platform).toBe('groq');
      expect(hit!.modelId).toBe('llama-3.3-70b');
      expect(hit!.keyId).toBe(7);
      expect(hit!.promptTokens).toBe(10);
      expect(hit!.completionTokens).toBe(5);
    });

    it('overwrites and refreshes an existing entry', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      store(key, 'first');
      store(key, 'second');
      expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('second');
      expect(getCacheStats().entries).toBe(1);
    });

    it('skips an unserializable body rather than throwing', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      const circular: any = {};
      circular.self = circular;
      storeCachedResponse(key, {
        body: circular, platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      });
      expect(getCacheStats().entries).toBe(0);
      expect(getCachedResponse(key)).toBeNull();
    });
  });

  describe('TTL expiry', () => {
    it('treats an entry older than the TTL as a miss and deletes it', () => {
      process.env.RESPONSE_CACHE_TTL_SECONDS = '1';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      const t0 = 1_000_000;
      store(key, 'stale', t0);
      // 2s later, past the 1s TTL → miss.
      expect(getCachedResponse(key, t0 + 2000)).toBeNull();
      // The expired entry was purged.
      expect(getCacheStats().entries).toBe(0);
    });

    it('serves an entry still within the TTL', () => {
      process.env.RESPONSE_CACHE_TTL_SECONDS = '60';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      const t0 = 1_000_000;
      store(key, 'fresh', t0);
      expect(getCachedResponse(key, t0 + 30_000)).not.toBeNull();
    });
  });

  describe('hit accounting', () => {
    it('counts hits and tallies saved tokens', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
      store(key, 'answer');
      getCachedResponse(key);
      getCachedResponse(key);
      getCachedResponse(key);
      const stats = getCacheStats();
      expect(stats.entries).toBe(1);
      expect(stats.totalHits).toBe(3);
      expect(stats.savedPromptTokens).toBe(30); // 3 hits × 10
      expect(stats.savedCompletionTokens).toBe(15); // 3 hits × 5
    });
  });

  describe('entry cap eviction (LRU)', () => {
    it('evicts the oldest entries past RESPONSE_CACHE_MAX_ENTRIES', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';
      // Realistic, increasing timestamps so reads stay inside the default TTL;
      // the point under test is eviction by count, not TTL expiry.
      const base = 1_700_000_000_000;
      const keys = ['a', 'b', 'c', 'd', 'e'].map(t =>
        computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));
      keys.forEach((k, i) => store(k, `v${i}`, base + i));

      expect(getCacheStats().entries).toBe(3);
      // Two oldest gone, three newest survive.
      expect(getCachedResponse(keys[0]!, base + 10)).toBeNull();
      expect(getCachedResponse(keys[1]!, base + 10)).toBeNull();
      expect(getCachedResponse(keys[4]!, base + 10)).not.toBeNull();
    });

    it('spares a recently-read entry from eviction (true LRU, not FIFO)', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';
      const base = 1_700_000_000_000;
      const keys = ['a', 'b', 'c'].map(t =>
        computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));
      keys.forEach((k, i) => store(k, `v${i}`, base + i)); // a,b,c
      // Touch 'a' so it becomes most-recently-used; 'b' is now the coldest.
      expect(getCachedResponse(keys[0]!, base + 5)).not.toBeNull();
      // Insert a 4th → evicts the LRU, which is now 'b', NOT 'a'.
      const d = computeCacheKey({ model: 'auto', messages: [msg('user', 'd')] });
      store(d, 'v3', base + 6);
      expect(getCachedResponse(keys[0]!, base + 7)).not.toBeNull(); // 'a' survived
      expect(getCachedResponse(keys[1]!, base + 7)).toBeNull();     // 'b' evicted
    });
  });

  describe('clearCache', () => {
    it('removes every entry', () => {
      store(computeCacheKey({ model: 'auto', messages: [msg('user', 'x')] }), 'x');
      store(computeCacheKey({ model: 'auto', messages: [msg('user', 'y')] }), 'y');
      expect(clearCache()).toBe(2);
      expect(getCacheStats().entries).toBe(0);
    });
  });

  describe('temperature guard', () => {
    it('caches omitted temperature', () => {
      expect(isCacheableTemperature(undefined)).toBe(true);
    });
    it('respects RESPONSE_CACHE_MAX_TEMPERATURE', () => {
      process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.5';
      expect(isCacheableTemperature(0.2)).toBe(true);
      expect(isCacheableTemperature(0.5)).toBe(true);
      expect(isCacheableTemperature(0.9)).toBe(false);
    });
  });

  describe('directive parsing', () => {
    it('parses off/on aliases', () => {
      expect(parseCacheDirective('off')).toBe('off');
      expect(parseCacheDirective('bypass')).toBe('off');
      expect(parseCacheDirective('on')).toBe('on');
      expect(parseCacheDirective('force')).toBe('on');
      expect(parseCacheDirective(undefined)).toBe('default');
    });

    it('treats Cache-Control: no-store as off', () => {
      expect(parseCacheDirective(undefined, 'no-store')).toBe('off');
    });

    it('takes the first value of a repeated header', () => {
      expect(parseCacheDirective(['off', 'on'])).toBe('off');
    });
  });

  describe('cacheActive (env switch)', () => {
    it('off directive always wins; on directive forces; default follows env', () => {
      delete process.env.RESPONSE_CACHE;
      expect(isCacheEnabled()).toBe(false);
      expect(cacheActive('off')).toBe(false);
      expect(cacheActive('on')).toBe(true);
      expect(cacheActive('default')).toBe(false);
      process.env.RESPONSE_CACHE = 'on';
      expect(cacheActive('default')).toBe(true);
      expect(cacheActive('off')).toBe(false);
    });
  });

  describe('streaming cache', () => {
    const frames = (text: string) => [
      `data: ${JSON.stringify({ choices: [{ delta: { content: text }, finish_reason: null }] })}\n\n`,
      'data: [DONE]\n\n',
    ];

    it('returns null on a streaming miss', () => {
      expect(getCachedStreamResponse('does-not-exist')).toBeNull();
    });

    it('replays the exact SSE frames on a streaming hit', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stream me')] });
      storeCachedStreamResponse(key, {
        frames: frames('streamed answer'),
        platform: 'groq',
        modelId: 'llama-3.3-70b',
        keyId: 7,
        promptTokens: 10,
        completionTokens: 5,
      });
      const hit = getCachedStreamResponse(key);
      expect(hit).not.toBeNull();
      expect(hit!.sse).toBe(frames('streamed answer').join(''));
      expect(hit!.platform).toBe('groq');
      expect(hit!.modelId).toBe('llama-3.3-70b');
    });

    it('keeps JSON and streaming entries separate for the same key', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'both')] });
      store(key, 'json answer');
      storeCachedStreamResponse(key, {
        frames: frames('stream answer'),
        platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      });
      // The JSON store still serves its entry; the stream store its own.
      expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('json answer');
      expect(getCachedStreamResponse(key)!.sse).toBe(frames('stream answer').join(''));
    });

    it('rejects an empty frame list (nothing to replay)', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'empty')] });
      storeCachedStreamResponse(key, {
        frames: [],
        platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      });
      expect(getCachedStreamResponse(key)).toBeNull();
    });

    it('counts streaming hits in the aggregated stats and clearCache clears both', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stats')] });
      storeCachedStreamResponse(key, {
        frames: frames('a'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 10, completionTokens: 5,
      });
      getCachedStreamResponse(key);
      getCachedStreamResponse(key);
      const stats = getCacheStats();
      expect(stats.entries).toBe(1);
      expect(stats.totalHits).toBe(2);
      expect(stats.savedPromptTokens).toBe(20); // 2 hits × 10
      expect(clearCache()).toBe(1);
      expect(getCacheStats().entries).toBe(0);
    });

    it('counts streaming lookups in lookupHits / lookupMisses (dashboard hit rate)', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'rate')] });
      // Two misses: an absent key, then an expired entry.
      expect(getCachedStreamResponse(key)).toBeNull();
      storeCachedStreamResponse(key, {
        frames: frames('a'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      }, 1_000);
      expect(getCachedStreamResponse(key, 1_000 + 2 * 3600 * 1000)).toBeNull(); // past the 1h TTL
      // Then a hit on a fresh entry.
      storeCachedStreamResponse(key, {
        frames: frames('a'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      }, 2_000);
      expect(getCachedStreamResponse(key, 2_100)).not.toBeNull();

      const stats = getCacheStats();
      expect(stats.lookupMisses).toBe(2);
      expect(stats.lookupHits).toBe(1);
      expect(stats.hitRate).toBeCloseTo(1 / 3);
    });

    it('skips a stream that outgrew the replay byte ceiling', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'huge')] });
      const huge = 'x'.repeat(STREAM_CACHE_MAX_BYTES + 1);
      storeCachedStreamResponse(key, {
        frames: [`data: ${huge}\n\n`],
        platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      });
      expect(getCachedStreamResponse(key)).toBeNull();
      expect(getCacheStats().entries).toBe(0);
    });

    it('shares one RESPONSE_CACHE_MAX_ENTRIES budget with the JSON store', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';
      const base = 1_700_000_000_000;
      const k = (t: string) => computeCacheKey({ model: 'auto', messages: [msg('user', t)] });
      store(k('j1'), 'v1', base + 1);
      store(k('j2'), 'v2', base + 2);
      storeCachedStreamResponse(k('s1'), {
        frames: frames('s1'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      }, base + 3);
      expect(getCacheStats().entries).toBe(3); // full, counting both stores

      // A 4th entry (either kind) evicts the coldest across BOTH stores, so
      // the total never exceeds the documented cap.
      storeCachedStreamResponse(k('s2'), {
        frames: frames('s2'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
      }, base + 4);
      expect(getCacheStats().entries).toBe(3);
      expect(getCachedResponse(k('j1'), base + 5)).toBeNull();           // oldest overall, evicted
      expect(getCachedResponse(k('j2'), base + 5)).not.toBeNull();
      expect(getCachedStreamResponse(k('s1'), base + 5)).not.toBeNull();
      expect(getCachedStreamResponse(k('s2'), base + 5)).not.toBeNull();

      // And a JSON insert can evict a colder STREAM entry, not just its own kind.
      store(k('j3'), 'v3', base + 6);
      store(k('j4'), 'v4', base + 7);
      expect(getCacheStats().entries).toBe(3);
      expect(getCachedStreamResponse(k('s1'), base + 8)).toBeNull();
    });
  });

  describe('settings-backed toggle (no restart)', () => {
    it('the stored setting enables the cache even with the env var unset', () => {
      delete process.env.RESPONSE_CACHE;
      expect(isCacheEnabled()).toBe(false);
      setSetting(CACHE_ENABLED_SETTING, '1');
      expect(isCacheEnabled()).toBe(true);
    });

    it('the stored setting wins over the env var', () => {
      process.env.RESPONSE_CACHE = 'on';
      setSetting(CACHE_ENABLED_SETTING, '0');
      expect(isCacheEnabled()).toBe(false); // explicit off beats env on
      setSetting(CACHE_ENABLED_SETTING, ''); // cleared → fall back to env
      expect(isCacheEnabled()).toBe(true);
    });
  });

  describe('hit rate', () => {
    it('counts every lookup, so an eviction cannot drag the rate down', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '1';
      const a = computeCacheKey({ model: 'auto', messages: [msg('user', 'a')] });
      const b = computeCacheKey({ model: 'auto', messages: [msg('user', 'b')] });

      store(a, 'answer a');
      expect(getCachedResponse(a)).not.toBeNull(); // 1 hit
      // Storing b evicts a, taking a's hit_count with it.
      store(b, 'answer b');
      expect(getCachedResponse(b)).not.toBeNull(); // 2 hits

      const stats = getCacheStats();
      expect(stats.lookupHits).toBe(2);
      expect(stats.lookupMisses).toBe(0);
      expect(stats.hitRate).toBe(1);
      // The entry-attributed count did shrink with the eviction; that is the
      // savings figure, and precisely why the rate must not be derived from it.
      expect(stats.totalHits).toBe(1);
    });

    it('counts misses, including a lookup that found an expired entry', () => {
      process.env.RESPONSE_CACHE_TTL_SECONDS = '60';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'rate')] });
      const t0 = 5_000_000;

      expect(getCachedResponse(key, t0)).toBeNull(); // cold miss
      store(key, 'answer', t0);
      expect(getCachedResponse(key, t0 + 1_000)).not.toBeNull(); // hit
      expect(getCachedResponse(key, t0 + 120_000)).toBeNull(); // expired: a miss

      const stats = getCacheStats();
      expect(stats.lookupHits).toBe(1);
      expect(stats.lookupMisses).toBe(2);
      expect(stats.hitRate).toBeCloseTo(1 / 3, 6);
    });

    it('reports 0 before any lookup rather than dividing by zero', () => {
      const stats = getCacheStats();
      expect(stats.lookupHits).toBe(0);
      expect(stats.lookupMisses).toBe(0);
      expect(stats.hitRate).toBe(0);
    });

    it('resets the tallies when the cache is flushed', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'flush-rate')] });
      store(key, 'answer');
      getCachedResponse(key);
      expect(getCacheStats().hitRate).toBe(1);

      clearCache();
      // A 100% hit rate beside zero entries would be nonsense.
      expect(getCacheStats().hitRate).toBe(0);
    });
  });

  // Persistence is tied to the cache master switch, so these have to turn it on.
  // "Restart" means __resetMemoryForTests(): it drops the in-memory LRU and
  // leaves the table alone, which is what a process restart does. clearCache()
  // is the user-facing flush and now wipes SQLite too, so it cannot stand in.
  describe('SQLite persistence', () => {
    beforeEach(() => {
      process.env.RESPONSE_CACHE = 'true';
    });

    const rowCount = (): number =>
      (getDb().prepare('SELECT COUNT(*) AS n FROM response_cache').get() as { n: number }).n;

    it('survives a restart: store -> reload from DB -> hit', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'persisted')] });
      store(key, 'persisted answer');
      expect(getCacheStats().entries).toBe(1);

      __resetMemoryForTests();
      expect(getCacheStats().entries).toBe(0);
      loadCacheFromDb();

      const hit = getCachedResponse(key);
      expect(hit).not.toBeNull();
      expect((hit!.body as any).choices[0].message.content).toBe('persisted answer');
      expect(hit!.platform).toBe('groq');
      expect(hit!.promptTokens).toBe(10);
    });

    it('persists nothing while the cache is off', () => {
      process.env.RESPONSE_CACHE = 'false';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'unpersisted')] });
      store(key, 'answer');
      __flushPersistenceForTests();
      expect(rowCount()).toBe(0);
    });

    it('persists nothing when RESPONSE_CACHE_PERSIST is off', () => {
      process.env.RESPONSE_CACHE_PERSIST = 'false';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'memory-only')] });
      store(key, 'answer');
      __flushPersistenceForTests();
      expect(rowCount()).toBe(0);
      // Still cached in memory: only the disk half is disabled.
      expect(getCachedResponse(key)).not.toBeNull();
    });

    it('carries the rolling hit count across a restart', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'counted')] });
      store(key, 'answer');
      getCachedResponse(key);
      getCachedResponse(key);
      expect(getCacheStats().totalHits).toBe(2);

      __resetMemoryForTests();
      loadCacheFromDb();
      // The hits reached the row, so the savings numbers do not reset to zero.
      expect(getCacheStats().totalHits).toBe(2);
      getCachedResponse(key);
      expect(getCacheStats().totalHits).toBe(3);
    });

    it('does not reload entries expired before startup', () => {
      process.env.RESPONSE_CACHE_TTL_SECONDS = '60';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stale-persisted')] });
      const t0 = 1_000_000;
      store(key, 'stale', t0);

      __resetMemoryForTests();
      // 10 minutes later (past the 60s TTL): startup purge drops the row.
      loadCacheFromDb(t0 + 600_000);
      expect(getCachedResponse(key, t0 + 600_000)).toBeNull();
      expect(getCacheStats().entries).toBe(0);
      expect(rowCount()).toBe(0);
    });

    it('respects RESPONSE_CACHE_MAX_ENTRIES when reloading', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '2';
      const base = 1_700_000_000_000;
      const keys = ['p1', 'p2', 'p3'].map(t =>
        computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));
      // Stored with the cap raised so all three reach the table, then reloaded
      // under the cap — the load path has its own bound, separate from the LRU.
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '10';
      keys.forEach((k, i) => store(k, `v${i}`, base + i));
      __flushPersistenceForTests();
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '2';

      __resetMemoryForTests();
      loadCacheFromDb(base + 10);
      // Newest two win; the oldest row is dropped past the cap.
      expect(getCacheStats().entries).toBe(2);
      expect(getCachedResponse(keys[0]!, base + 10)).toBeNull();
      expect(getCachedResponse(keys[2]!, base + 10)).not.toBeNull();
    });

    it('drops the row when the LRU evicts the entry', () => {
      process.env.RESPONSE_CACHE_MAX_ENTRIES = '1';
      const first = computeCacheKey({ model: 'auto', messages: [msg('user', 'evicted')] });
      const second = computeCacheKey({ model: 'auto', messages: [msg('user', 'survivor')] });
      store(first, 'gone');
      store(second, 'kept');
      __flushPersistenceForTests();

      expect(rowCount()).toBe(1);
      __resetMemoryForTests();
      loadCacheFromDb();
      // The evicted entry must not come back from disk.
      expect(getCachedResponse(first)).toBeNull();
      expect(getCachedResponse(second)).not.toBeNull();
    });

    it('drops the row when a read finds the entry expired', () => {
      process.env.RESPONSE_CACHE_TTL_SECONDS = '60';
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'ttl-row')] });
      const t0 = 2_000_000;
      store(key, 'stale', t0);
      __flushPersistenceForTests();
      expect(rowCount()).toBe(1);

      expect(getCachedResponse(key, t0 + 120_000)).toBeNull();
      __flushPersistenceForTests();
      expect(rowCount()).toBe(0);
    });

    describe('batched hit-count flush', () => {
      const hitRow = (key: string) => getDb()
        .prepare('SELECT hit_count, last_hit_at_ms FROM response_cache WHERE cache_key = ?')
        .get(key) as { hit_count: number; last_hit_at_ms: number | null } | undefined;

      beforeEach(() => { vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); });
      afterEach(() => { vi.useRealTimers(); });

      it('writes many hits in one debounced pass', () => {
        const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'batched')] });
        const t0 = 5_000_000;
        store(key, 'answer', t0);
        __flushPersistenceForTests();
        getCachedResponse(key, t0 + 1);
        getCachedResponse(key, t0 + 2);
        getCachedResponse(key, t0 + 3);
        __flushPersistenceForTests();
        // Nothing written per hit: the row still holds the stored value.
        expect(hitRow(key)!.hit_count).toBe(0);

        vi.advanceTimersByTime(2_000);
        __flushPersistenceForTests();
        expect(hitRow(key)).toEqual({ hit_count: 3, last_hit_at_ms: t0 + 3 });
      });

      it('clearCache() drops pending counts so they never land on a re-stored entry', () => {
        const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'cleared-pending')] });
        store(key, 'old answer');
        __flushPersistenceForTests();
        getCachedResponse(key);
        getCachedResponse(key);

        clearCache();
        store(key, 'fresh answer');
        __flushPersistenceForTests();
        vi.advanceTimersByTime(2_000);
        __flushPersistenceForTests();

        expect(hitRow(key)!.hit_count).toBe(0);
        expect(getCacheStats().totalHits).toBe(0);
      });

      it('a re-store drops the old entry\'s pending count', () => {
        const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'restored-pending')] });
        store(key, 'old answer');
        __flushPersistenceForTests();
        getCachedResponse(key);
        getCachedResponse(key);

        // Overwrite before the flush timer fires: the fresh row starts at 0.
        store(key, 'new answer');
        __flushPersistenceForTests();
        vi.advanceTimersByTime(2_000);
        __flushPersistenceForTests();
        expect(hitRow(key)!.hit_count).toBe(0);

        // Hits on the new entry still flush normally.
        getCachedResponse(key);
        vi.advanceTimersByTime(2_000);
        __flushPersistenceForTests();
        expect(hitRow(key)!.hit_count).toBe(1);
      });
    });

    it('clearCache() purges SQLite too, so a restart does not resurrect entries', () => {
      const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'flushed')] });
      store(key, 'answer');
      __flushPersistenceForTests();
      expect(rowCount()).toBe(1);

      expect(clearCache()).toBe(1);
      expect(rowCount()).toBe(0);
      loadCacheFromDb();
      expect(getCachedResponse(key)).toBeNull();
    });
  });
});