/** * One-shot real LLM smoke test for tabular batch extraction parsing. * Usage: npx tsx scripts/tabularExtractionSmoke.ts */ import "dotenv/config"; import { streamChatWithTools } from "../src/lib/llm"; import { DEFAULT_TABULAR_MODEL } from "../src/lib/llm/models"; import { buildExtractionBatchUser } from "../src/prompts/tabular/extraction-batch-user"; import { TabularJsonStreamParser } from "../src/lib/tabularJsonStreamParser"; function debugLog( message: string, data: Record, ) { console.log(JSON.stringify({ smoke: { message, data } })); } const SAMPLE_DOC = ` COMMERCIAL PROPOSAL — SUPPLY & ERECTION 1. Document Type: This is a commercial quotation / proposal for pre-engineered building supply and erection. 2. Zamil Role: Zamil Steel acts as the Seller / Supplier for the materials listed herein. 3. Contract Type: This proposal is issued on a lump-sum fixed price basis. 4. Price Basis: All prices are firm and fixed for 90 days from proposal date. 5. Price Escalation: No price escalation applies unless expressly agreed in writing. `.trim(); const COLUMNS = [ { index: 0, name: "Document Type", prompt: "Identify the document type.", format: "text", }, { index: 1, name: "Zamil Role Stated", prompt: "State Zamil's role as described in the document.", format: "text", }, { index: 2, name: "Contract Type", prompt: "Extract the contract/pricing type (e.g. lump sum, unit rate).", format: "text", }, { index: 3, name: "Price Basis", prompt: "Extract whether pricing is fixed, firm, or subject to adjustment.", format: "text", }, { index: 4, name: "Price Escalation", prompt: "Extract any price escalation or variation clause.", format: "text", }, ]; async function main() { const model = process.env.TABULAR_SMOKE_MODEL || DEFAULT_TABULAR_MODEL; const batch = buildExtractionBatchUser({ filename: "PEB-sample-proposal.docx", documentText: SAMPLE_DOC, columns: COLUMNS, }); debugLog("starting real LLM tabular smoke test", { model, columnCount: COLUMNS.length, columnIndices: COLUMNS.map((c) => c.index), }); const parser = new TabularJsonStreamParser(); const received: Record< number, { summary: string; flag: string; reasoning: string } > = {}; let streamError: string | null = null; let fullText = ""; try { const result = await streamChatWithTools({ model, systemPrompt: batch.system, messages: [{ role: "user", content: batch.user }], tools: [], apiKeys: { openai: process.env.OPENAI_API_KEY ?? null, gemini: process.env.GEMINI_API_KEY ?? null, }, callbacks: { onContentDelta: (delta) => { fullText += delta; for (const obj of parser.append(delta)) { const rawIndex = obj.column_index; const idx = typeof rawIndex === "number" ? rawIndex : typeof rawIndex === "string" ? Number(rawIndex) : NaN; if (!Number.isFinite(idx)) continue; received[idx] = { summary: String(obj.summary ?? "").trim(), flag: String(obj.flag ?? "grey"), reasoning: String(obj.reasoning ?? "").slice( 0, 120, ), }; } }, }, }); fullText = result.fullText || fullText; } catch (err) { streamError = err instanceof Error ? err.message : String(err); debugLog("provider stream failed", { model, error: streamError }); } for (const obj of parser.flush()) { const rawIndex = obj.column_index; const idx = typeof rawIndex === "number" ? rawIndex : typeof rawIndex === "string" ? Number(rawIndex) : NaN; if (!Number.isFinite(idx)) continue; received[idx] = { summary: String(obj.summary ?? "").trim(), flag: String(obj.flag ?? "grey"), reasoning: String(obj.reasoning ?? "").slice(0, 120), }; } const expected = COLUMNS.map((c) => c.index); const got = Object.keys(received).map(Number).sort((a, b) => a - b); const missing = expected.filter((i) => !(i in received)); const report = { model, streamError, expectedColumns: expected, receivedColumns: got, missingColumns: missing, parseFailures: parser.parseFailureCount, remainderLength: parser.remainder.length, remainderPreview: parser.remainder.slice(0, 300), responseLength: fullText.length, responsePreview: fullText.slice(0, 400), cells: COLUMNS.map((col) => ({ index: col.index, name: col.name, status: col.index in received ? "parsed" : "missing", summary: received[col.index]?.summary?.slice(0, 120) ?? null, flag: received[col.index]?.flag ?? null, })), }; debugLog("real LLM smoke test finished", report); console.log("\n=== TABULAR LLM SMOKE TEST REPORT ==="); console.log(JSON.stringify(report, null, 2)); if (streamError || missing.length > 0 || parser.parseFailureCount > 0) { process.exitCode = 1; } } main().catch((err) => { debugLog("smoke script crashed", { error: err instanceof Error ? err.message : String(err), }); console.error(err); process.exit(1); });