File size: 3,233 Bytes
dcedb25 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 | // Structured-output enforcement for the fallback chain (#514 follow-up).
//
// Forwarding response_format is necessary but not sufficient: a free-tier
// model that doesn't understand JSON mode often just answers in prose (or
// wraps the JSON in a markdown fence) and the request LOOKS successful — the
// worst failure mode, because the client asked for machine-readable output
// and got an essay. On non-streaming responses the gateway can check, heal,
// or fail over:
//
// 1. content parses as JSON → pass through untouched
// 2. content wraps JSON in ```fences``` or leading/trailing prose
// → heal: replace content with the JSON
// 3. nothing parseable → retryable failure (skipBench: the
// provider is healthy, the MODEL ignored the format — no cooldown, no
// penalty; the chain just tries the next candidate)
//
// Streaming responses used to skip enforcement entirely (bytes already on
// the wire). Since #933 the proxy's stream path buffers structured-output
// requests in a 'json' hold mode — headers are held until the stream ends,
// so the same check/heal/fail-over applies there too. The response cache
// remains non-stream only.
export type JsonEnforcement =
| { ok: true; content: string; healed: boolean }
| { ok: false };
/** Match a fenced code block (```json ... ``` or bare ``` ... ```). */
const FENCE_RE = /```(?:json)?\s*\n?([\s\S]*?)```/i;
function parses(text: string): boolean {
try {
JSON.parse(text);
return true;
} catch {
return false;
}
}
/**
* Candidate slices from the first '{' to the last '}' and from the first '['
* to the last ']', longest first. Both bracket types must be tried
* independently: picking whichever opener appears FIRST turned
* `According to [1], here is the JSON: {"city":"Paris"}` into the citation
* marker `[1]` — a valid parse of the wrong value, silently delivered as the
* structured output. Longest-first means an incidental bracket token in prose
* can never shadow the real payload next to it.
*/
function candidateJsonSlices(text: string): string[] {
const out: string[] = [];
for (const [open, close] of [['{', '}'], ['[', ']']] as const) {
const first = text.indexOf(open);
const last = text.lastIndexOf(close);
if (first !== -1 && last > first) out.push(text.slice(first, last + 1));
}
return out.sort((a, b) => b.length - a.length);
}
export function enforceJsonContent(content: string): JsonEnforcement {
const trimmed = content.trim();
if (trimmed.length === 0) return { ok: false };
if (parses(trimmed)) return { ok: true, content: trimmed === content ? content : trimmed, healed: trimmed !== content };
// Fenced block: the most common "almost right" shape free models produce.
const fence = FENCE_RE.exec(trimmed);
if (fence) {
const inner = fence[1].trim();
if (parses(inner)) return { ok: true, content: inner, healed: true };
}
// Leading/trailing prose around one JSON value ("Here is your JSON: {...}").
for (const slice of candidateJsonSlices(trimmed)) {
if (parses(slice)) return { ok: true, content: slice, healed: true };
}
return { ok: false };
}
|