File size: 5,085 Bytes
bc575bc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | import type { AssistantMessage } from "@earendil-works/pi-ai";
import { describe, expect, it } from "vitest";
import {
collectCacheMisses,
computeCacheWaste,
detectCacheMiss,
type ModelPriceSource,
} from "../src/core/cache-stats.ts";
import type { SessionEntry } from "../src/core/session-manager.ts";
const zeroCost = { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 };
const models: ModelPriceSource = {
// $/million tokens; used as cache-read price fallback on full-miss turns
getModel: () => ({ cost: { cacheRead: 0.3 } }),
};
function assistant(options: {
input?: number;
cacheRead?: number;
cacheWrite?: number;
cost?: Partial<typeof zeroCost>;
model?: string;
timestamp?: number;
}): AssistantMessage {
return {
role: "assistant",
content: [],
api: "anthropic-messages",
provider: "test",
model: options.model ?? "test-model",
usage: {
input: options.input ?? 0,
output: 10,
cacheRead: options.cacheRead ?? 0,
cacheWrite: options.cacheWrite ?? 0,
totalTokens: 0,
cost: { ...zeroCost, ...options.cost },
},
stopReason: "stop",
timestamp: options.timestamp ?? 0,
} as AssistantMessage;
}
function entry(message: AssistantMessage): SessionEntry {
return { type: "message", id: "x", parentId: null, timestamp: "", message } as SessionEntry;
}
// Turn 1: fresh 100k cache write at $3.75/M
const turn1 = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, timestamp: 0 });
// Turn 2: healthy, everything read back at $0.30/M
const turn2 = assistant({
cacheRead: 100_000,
cacheWrite: 5_000,
cost: { cacheRead: 0.03, cacheWrite: 0.019 },
timestamp: 60_000,
});
describe("computeCacheWaste", () => {
it("accumulates missed tokens and cost across turns", () => {
// Turn 3: full miss, previous 105k prompt re-billed at $3.75/M write
const turn3 = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
const totals = computeCacheWaste([entry(turn1), entry(turn2), entry(turn3)], models);
expect(totals.missedTokens).toBe(105_000);
// 105k at ($3.75 - $0.30)/M
expect(totals.missedCost).toBeCloseTo(0.36225, 5);
});
it("counts nothing for healthy sessions", () => {
const totals = computeCacheWaste([entry(turn1), entry(turn2)], models);
expect(totals.missedTokens).toBe(0);
expect(totals.missedCost).toBe(0);
});
it("skips the turn after a compaction reset", () => {
const reset = { type: "compaction", id: "c", parentId: null, timestamp: "" } as SessionEntry;
const afterReset = assistant({ cacheWrite: 20_000, cost: { cacheWrite: 0.075 } });
const totals = computeCacheWaste([entry(turn1), reset, entry(afterReset)], models);
expect(totals.missedTokens).toBe(0);
});
it("counts misses caused by model switches", () => {
const otherModel = assistant({ cacheWrite: 100_000, cost: { cacheWrite: 0.375 }, model: "other-model" });
const totals = computeCacheWaste([entry(turn1), entry(otherModel)], models);
expect(totals.missedTokens).toBe(100_000);
expect(totals.missCount).toBe(1);
});
it("skips providers that report no cache activity", () => {
const a = assistant({ input: 100_000 });
const b = assistant({ input: 110_000 });
const totals = computeCacheWaste([entry(a), entry(b)], models);
expect(totals.missedTokens).toBe(0);
});
});
describe("collectCacheMisses", () => {
it("maps counted misses to their assistant messages by reference", () => {
const missTurn = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 120_000 });
const misses = collectCacheMisses([entry(turn1), entry(turn2), entry(missTurn)], models);
expect(misses.size).toBe(1);
expect(misses.get(missTurn)?.missedTokens).toBe(105_000);
});
});
describe("detectCacheMiss", () => {
it("detects a miss on a just-completed message with idle time", () => {
const missMessage = assistant({ cacheWrite: 110_000, cost: { cacheWrite: 0.4125 }, timestamp: 600_000 });
const miss = detectCacheMiss([entry(turn1), entry(turn2)], missMessage, models);
expect(miss).toBeDefined();
expect(miss?.missedTokens).toBe(105_000);
expect(miss?.missedCost).toBeCloseTo(0.36225, 5);
// 600s - 60s since the previous request
expect(miss?.idleMs).toBe(540_000);
expect(miss?.modelChanged).toBe(false);
});
it("flags model switches on detected misses", () => {
const otherModel = assistant({
cacheWrite: 110_000,
cost: { cacheWrite: 0.4125 },
model: "other-model",
timestamp: 120_000,
});
const miss = detectCacheMiss([entry(turn1), entry(turn2)], otherModel, models);
expect(miss?.missedTokens).toBe(105_000);
expect(miss?.modelChanged).toBe(true);
});
it("returns undefined for healthy turns", () => {
const healthy = assistant({
cacheRead: 105_000,
cacheWrite: 2_000,
cost: { cacheRead: 0.0315, cacheWrite: 0.0075 },
timestamp: 120_000,
});
expect(detectCacheMiss([entry(turn1), entry(turn2)], healthy, models)).toBeUndefined();
});
it("returns undefined for the first turn of a session", () => {
expect(detectCacheMiss([], turn1, models)).toBeUndefined();
});
});
|