Spaces:
Sleeping
Sleeping
| /** | |
| * One-shot real LLM smoke test for tabular batch extraction parsing. | |
| * Usage: npx tsx scripts/tabularExtractionSmoke.ts | |
| */ | |
| import "dotenv/config"; | |
| import { streamChatWithTools } from "../src/lib/llm"; | |
| import { DEFAULT_TABULAR_MODEL } from "../src/lib/llm/models"; | |
| import { buildExtractionBatchUser } from "../src/prompts/tabular/extraction-batch-user"; | |
| import { TabularJsonStreamParser } from "../src/lib/tabularJsonStreamParser"; | |
| function debugLog( | |
| message: string, | |
| data: Record<string, unknown>, | |
| ) { | |
| console.log(JSON.stringify({ smoke: { message, data } })); | |
| } | |
| const SAMPLE_DOC = ` | |
| COMMERCIAL PROPOSAL — SUPPLY & ERECTION | |
| 1. Document Type: This is a commercial quotation / proposal for pre-engineered building supply and erection. | |
| 2. Zamil Role: Zamil Steel acts as the Seller / Supplier for the materials listed herein. | |
| 3. Contract Type: This proposal is issued on a lump-sum fixed price basis. | |
| 4. Price Basis: All prices are firm and fixed for 90 days from proposal date. | |
| 5. Price Escalation: No price escalation applies unless expressly agreed in writing. | |
| `.trim(); | |
| const COLUMNS = [ | |
| { | |
| index: 0, | |
| name: "Document Type", | |
| prompt: "Identify the document type.", | |
| format: "text", | |
| }, | |
| { | |
| index: 1, | |
| name: "Zamil Role Stated", | |
| prompt: "State Zamil's role as described in the document.", | |
| format: "text", | |
| }, | |
| { | |
| index: 2, | |
| name: "Contract Type", | |
| prompt: "Extract the contract/pricing type (e.g. lump sum, unit rate).", | |
| format: "text", | |
| }, | |
| { | |
| index: 3, | |
| name: "Price Basis", | |
| prompt: "Extract whether pricing is fixed, firm, or subject to adjustment.", | |
| format: "text", | |
| }, | |
| { | |
| index: 4, | |
| name: "Price Escalation", | |
| prompt: "Extract any price escalation or variation clause.", | |
| format: "text", | |
| }, | |
| ]; | |
| async function main() { | |
| const model = process.env.TABULAR_SMOKE_MODEL || DEFAULT_TABULAR_MODEL; | |
| const batch = buildExtractionBatchUser({ | |
| filename: "PEB-sample-proposal.docx", | |
| documentText: SAMPLE_DOC, | |
| columns: COLUMNS, | |
| }); | |
| debugLog("starting real LLM tabular smoke test", { | |
| model, | |
| columnCount: COLUMNS.length, | |
| columnIndices: COLUMNS.map((c) => c.index), | |
| }); | |
| const parser = new TabularJsonStreamParser(); | |
| const received: Record< | |
| number, | |
| { summary: string; flag: string; reasoning: string } | |
| > = {}; | |
| let streamError: string | null = null; | |
| let fullText = ""; | |
| try { | |
| const result = await streamChatWithTools({ | |
| model, | |
| systemPrompt: batch.system, | |
| messages: [{ role: "user", content: batch.user }], | |
| tools: [], | |
| apiKeys: { | |
| openai: process.env.OPENAI_API_KEY ?? null, | |
| gemini: process.env.GEMINI_API_KEY ?? null, | |
| }, | |
| callbacks: { | |
| onContentDelta: (delta) => { | |
| fullText += delta; | |
| for (const obj of parser.append(delta)) { | |
| const rawIndex = obj.column_index; | |
| const idx = | |
| typeof rawIndex === "number" | |
| ? rawIndex | |
| : typeof rawIndex === "string" | |
| ? Number(rawIndex) | |
| : NaN; | |
| if (!Number.isFinite(idx)) continue; | |
| received[idx] = { | |
| summary: String(obj.summary ?? "").trim(), | |
| flag: String(obj.flag ?? "grey"), | |
| reasoning: String(obj.reasoning ?? "").slice( | |
| 0, | |
| 120, | |
| ), | |
| }; | |
| } | |
| }, | |
| }, | |
| }); | |
| fullText = result.fullText || fullText; | |
| } catch (err) { | |
| streamError = err instanceof Error ? err.message : String(err); | |
| debugLog("provider stream failed", { model, error: streamError }); | |
| } | |
| for (const obj of parser.flush()) { | |
| const rawIndex = obj.column_index; | |
| const idx = | |
| typeof rawIndex === "number" | |
| ? rawIndex | |
| : typeof rawIndex === "string" | |
| ? Number(rawIndex) | |
| : NaN; | |
| if (!Number.isFinite(idx)) continue; | |
| received[idx] = { | |
| summary: String(obj.summary ?? "").trim(), | |
| flag: String(obj.flag ?? "grey"), | |
| reasoning: String(obj.reasoning ?? "").slice(0, 120), | |
| }; | |
| } | |
| const expected = COLUMNS.map((c) => c.index); | |
| const got = Object.keys(received).map(Number).sort((a, b) => a - b); | |
| const missing = expected.filter((i) => !(i in received)); | |
| const report = { | |
| model, | |
| streamError, | |
| expectedColumns: expected, | |
| receivedColumns: got, | |
| missingColumns: missing, | |
| parseFailures: parser.parseFailureCount, | |
| remainderLength: parser.remainder.length, | |
| remainderPreview: parser.remainder.slice(0, 300), | |
| responseLength: fullText.length, | |
| responsePreview: fullText.slice(0, 400), | |
| cells: COLUMNS.map((col) => ({ | |
| index: col.index, | |
| name: col.name, | |
| status: col.index in received ? "parsed" : "missing", | |
| summary: received[col.index]?.summary?.slice(0, 120) ?? null, | |
| flag: received[col.index]?.flag ?? null, | |
| })), | |
| }; | |
| debugLog("real LLM smoke test finished", report); | |
| console.log("\n=== TABULAR LLM SMOKE TEST REPORT ==="); | |
| console.log(JSON.stringify(report, null, 2)); | |
| if (streamError || missing.length > 0 || parser.parseFailureCount > 0) { | |
| process.exitCode = 1; | |
| } | |
| } | |
| main().catch((err) => { | |
| debugLog("smoke script crashed", { | |
| error: err instanceof Error ? err.message : String(err), | |
| }); | |
| console.error(err); | |
| process.exit(1); | |
| }); | |