Blame for backend/scripts/tabularExtractionSmoke.ts

/**
 * One-shot real LLM smoke test for tabular batch extraction parsing.
 * Usage: npx tsx scripts/tabularExtractionSmoke.ts
 */
import "dotenv/config";
import { streamChatWithTools } from "../src/lib/llm";
import { DEFAULT_TABULAR_MODEL } from "../src/lib/llm/models";
import { buildExtractionBatchUser } from "../src/prompts/tabular/extraction-batch-user";
import { TabularJsonStreamParser } from "../src/lib/tabularJsonStreamParser";

function debugLog(
    message: string,
    data: Record<string, unknown>,
) {
    console.log(JSON.stringify({ smoke: { message, data } }));
}

const SAMPLE_DOC = `
COMMERCIAL PROPOSAL — SUPPLY & ERECTION

1. Document Type: This is a commercial quotation / proposal for pre-engineered building supply and erection.

2. Zamil Role: Zamil Steel acts as the Seller / Supplier for the materials listed herein.

3. Contract Type: This proposal is issued on a lump-sum fixed price basis.

4. Price Basis: All prices are firm and fixed for 90 days from proposal date.

5. Price Escalation: No price escalation applies unless expressly agreed in writing.
`.trim();

const COLUMNS = [
    {
        index: 0,
        name: "Document Type",
        prompt: "Identify the document type.",
        format: "text",
    },
    {
        index: 1,
        name: "Zamil Role Stated",
        prompt: "State Zamil's role as described in the document.",
        format: "text",
    },
    {
        index: 2,
        name: "Contract Type",
        prompt: "Extract the contract/pricing type (e.g. lump sum, unit rate).",
        format: "text",
    },
    {
        index: 3,
        name: "Price Basis",
        prompt: "Extract whether pricing is fixed, firm, or subject to adjustment.",
        format: "text",
    },
    {
        index: 4,
        name: "Price Escalation",
        prompt: "Extract any price escalation or variation clause.",
        format: "text",
    },
];

async function main() {
    const model = process.env.TABULAR_SMOKE_MODEL || DEFAULT_TABULAR_MODEL;
    const batch = buildExtractionBatchUser({
        filename: "PEB-sample-proposal.docx",
        documentText: SAMPLE_DOC,
        columns: COLUMNS,
    });

    debugLog("starting real LLM tabular smoke test", {
        model,
        columnCount: COLUMNS.length,
        columnIndices: COLUMNS.map((c) => c.index),
    });

    const parser = new TabularJsonStreamParser();
    const received: Record<
        number,
        { summary: string; flag: string; reasoning: string }
    > = {};
    let streamError: string | null = null;
    let fullText = "";

    try {
        const result = await streamChatWithTools({
            model,
            systemPrompt: batch.system,
            messages: [{ role: "user", content: batch.user }],
            tools: [],
            apiKeys: {
                openai: process.env.OPENAI_API_KEY ?? null,
                gemini: process.env.GEMINI_API_KEY ?? null,
            },
            callbacks: {
                onContentDelta: (delta) => {
                    fullText += delta;
                    for (const obj of parser.append(delta)) {
                        const rawIndex = obj.column_index;
                        const idx =
                            typeof rawIndex === "number"
                                ? rawIndex
                                : typeof rawIndex === "string"
                                  ? Number(rawIndex)
                                  : NaN;
                        if (!Number.isFinite(idx)) continue;
                        received[idx] = {
                            summary: String(obj.summary ?? "").trim(),
                            flag: String(obj.flag ?? "grey"),
                            reasoning: String(obj.reasoning ?? "").slice(
                                0,
                                120,
                            ),
                        };
                    }
                },
            },
        });
        fullText = result.fullText || fullText;
    } catch (err) {
        streamError = err instanceof Error ? err.message : String(err);
        debugLog("provider stream failed", { model, error: streamError });
    }

    for (const obj of parser.flush()) {
        const rawIndex = obj.column_index;
        const idx =
            typeof rawIndex === "number"
                ? rawIndex
                : typeof rawIndex === "string"
                  ? Number(rawIndex)
                  : NaN;
        if (!Number.isFinite(idx)) continue;
        received[idx] = {
            summary: String(obj.summary ?? "").trim(),
            flag: String(obj.flag ?? "grey"),
            reasoning: String(obj.reasoning ?? "").slice(0, 120),
        };
    }

    const expected = COLUMNS.map((c) => c.index);
    const got = Object.keys(received).map(Number).sort((a, b) => a - b);
    const missing = expected.filter((i) => !(i in received));

    const report = {
        model,
        streamError,
        expectedColumns: expected,
        receivedColumns: got,
        missingColumns: missing,
        parseFailures: parser.parseFailureCount,
        remainderLength: parser.remainder.length,
        remainderPreview: parser.remainder.slice(0, 300),
        responseLength: fullText.length,
        responsePreview: fullText.slice(0, 400),
        cells: COLUMNS.map((col) => ({
            index: col.index,
            name: col.name,
            status: col.index in received ? "parsed" : "missing",
            summary: received[col.index]?.summary?.slice(0, 120) ?? null,
            flag: received[col.index]?.flag ?? null,
        })),
    };

    debugLog("real LLM smoke test finished", report);
    console.log("\n=== TABULAR LLM SMOKE TEST REPORT ===");
    console.log(JSON.stringify(report, null, 2));

    if (streamError || missing.length > 0 || parser.parseFailureCount > 0) {
        process.exitCode = 1;
    }
}

main().catch((err) => {
    debugLog("smoke script crashed", {
        error: err instanceof Error ? err.message : String(err),
    });
    console.error(err);
    process.exit(1);
});