File size: 3,807 Bytes
68d7816 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 | import type { Plugin } from '#/plugin';
export interface LlmRequestTiming {
readonly requestBuildMs?: number;
readonly ttftMs: number;
readonly serverFirstTokenMs: number;
readonly streamDurationMs: number;
readonly serverDecodeMs: number;
readonly clientConsumeMs: number;
}
export interface TimingPlugin extends Plugin {
readonly name: 'timing';
timing(): LlmRequestTiming | undefined;
}
export function createTimingPlugin(input?: { now?: () => number }): TimingPlugin {
const now = input?.now ?? Date.now;
let current: LlmRequestTiming | undefined;
let lastEventAt: number | undefined;
let retryAnchor: { at: number; delayMs: number } | undefined;
let sentAt: number | undefined;
let attemptStartedAt: number | undefined;
let firstDeltaAt: number | undefined;
let lastHandledAt = 0;
let serverDecodeMs = 0;
let clientConsumeMs = 0;
const resetWindow = (): void => {
sentAt = undefined;
attemptStartedAt = undefined;
firstDeltaAt = undefined;
serverDecodeMs = 0;
clientConsumeMs = 0;
};
return {
name: 'timing',
timing: () => current,
connect(target) {
if (target.kind !== 'agent') return;
const mark = (): void => {
lastEventAt = now();
};
target.on('turn.started', () => {
retryAnchor = undefined;
lastEventAt = now();
});
target.on('tool.detached', mark);
target.on('tool.done', mark);
target.on('tool.failed', mark);
target.on('tool.aborted', mark);
target.on('llm.sent', () => {
const t = now();
attemptStartedAt =
retryAnchor === undefined ? lastEventAt : retryAnchor.at + retryAnchor.delayMs;
retryAnchor = undefined;
sentAt = t;
firstDeltaAt = undefined;
serverDecodeMs = 0;
clientConsumeMs = 0;
lastEventAt = t;
});
target.on('llm.streaming.part', () => {
const arrivedAt = now();
if (sentAt === undefined) return;
if (firstDeltaAt === undefined) {
firstDeltaAt = arrivedAt;
} else {
serverDecodeMs += arrivedAt - lastHandledAt;
}
const handledAt = now();
clientConsumeMs += handledAt - arrivedAt;
lastHandledAt = handledAt;
lastEventAt = handledAt;
});
target.on('llm.done', () => {
const t = now();
if (sentAt !== undefined && firstDeltaAt !== undefined) {
serverDecodeMs += t - lastHandledAt;
current = {
requestBuildMs:
attemptStartedAt === undefined
? undefined
: Math.max(0, sentAt - attemptStartedAt),
ttftMs: Math.max(0, firstDeltaAt - (attemptStartedAt ?? sentAt)),
serverFirstTokenMs: Math.max(0, firstDeltaAt - sentAt),
streamDurationMs: Math.max(0, t - firstDeltaAt),
serverDecodeMs: Math.max(0, serverDecodeMs),
clientConsumeMs: Math.max(0, clientConsumeMs),
};
}
resetWindow();
lastEventAt = t;
});
target.on('llm.retrying', (event) => {
const t = now();
if (event.type === 'llm.retrying') {
retryAnchor = { at: t, delayMs: event.delayMs };
}
resetWindow();
lastEventAt = t;
});
target.on('llm.recovering', () => {
const t = now();
retryAnchor = undefined;
resetWindow();
lastEventAt = t;
});
target.on('llm.failed.syntax', () => {
const t = now();
retryAnchor = undefined;
resetWindow();
lastEventAt = t;
});
target.on('llm.failed.remote', () => {
const t = now();
retryAnchor = undefined;
resetWindow();
lastEventAt = t;
});
},
};
}
|