import assert from "node:assert/strict"; import { describe, it } from "node:test"; import { analyzeUserPrompt, compareAdapters, findHarnessPolicy, findModelProfile, generateBoostedPrompt, inferTaskType, modelAdapterProfiles, runEvalCase, runLocalFixtureComparison, scorePrompt } from "../src/index.ts"; import type { JudgeRunner, PromptRunner } from "../src/index.ts"; describe("PromptBoost core", () => { it("infers task types from raw user prompts", () => { assert.equal( inferTaskType("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt."), "structured_extraction" ); assert.equal( inferTaskType("Finde den Bug in einem Modul, implementiere den Fix und liefere Tests."), "coding_patch" ); }); it("creates a stable task spec without dropping the user intent", () => { const raw = "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript."; const spec = analyzeUserPrompt(raw, { taskType: "long_document" }); assert.equal(spec.rawUserPrompt, raw); assert.equal(spec.taskType, "long_document"); assert.match(spec.goal, /PDF/); assert.ok(spec.deliverables.some((item) => item.includes("Markdown"))); assert.ok(spec.suggestedScaffolding.deliverables.some((item) => item.includes("document map"))); assert.ok(spec.constraints.some((item) => item.includes("Preserve the user's task intent"))); }); it("builds a harness-neutral boosted prompt with score and eval plan", () => { const boosted = generateBoostedPrompt( "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.", { taskType: "long_document", modelProfileId: "gpt-openai", harnessPolicyId: "blank" } ); assert.match(boosted.prompt, /Task Spec/); assert.match(boosted.prompt, /Adapter-Suggested Scaffolding/); assert.match(boosted.prompt, /Harness Policy/); assert.match(boosted.prompt, /Model Adapter/); assert.match(boosted.prompt, /Verification/); assert.match(boosted.prompt, /provider-neutral/); assert.equal(boosted.evalPlan.metadataFields.includes("model"), true); assert.ok(boosted.score.overall >= 75); assert.equal(boosted.score.source, "prompt_shape_heuristic"); }); it("keeps all ten model adapter profiles available", () => { assert.equal(modelAdapterProfiles.length, 10); assert.deepEqual( modelAdapterProfiles.map((profile) => profile.id), [ "gpt-openai", "claude", "gemini", "kimi", "glm", "qwen", "deepseek", "mistral", "minimax", "llama" ] ); assert.equal(modelAdapterProfiles.every((profile) => Boolean(profile.evidenceNote)), true); assert.equal(modelAdapterProfiles.every((profile) => profile.evidence !== "strong"), true); }); it("blocks empty prompts instead of scoring the generated scaffold as strong", () => { const boosted = generateBoostedPrompt(""); assert.equal(boosted.inputAdequacy.status, "blocked"); assert.equal(boosted.score.grade, "blocked"); assert.ok(boosted.score.overall <= 35); assert.equal(boosted.taskSpec.deliverables.length, 0); assert.ok(boosted.taskSpec.suggestedScaffolding.deliverables.length > 0); }); it("keeps very short summarize prompts generic instead of inventing long-document work", () => { const boosted = generateBoostedPrompt("Summarize."); assert.equal(boosted.taskSpec.taskType, "general"); assert.equal(boosted.taskSpec.deliverables.includes("document map"), false); assert.equal(boosted.inputAdequacy.status, "needs_clarification"); assert.ok(boosted.score.overall < 62); }); it("preserves non-ASCII terminal punctuation in goals", () => { const boosted = generateBoostedPrompt("请用中文总结这段内容。"); assert.equal(boosted.taskSpec.goal, "请用中文总结这段内容。"); assert.notEqual(boosted.inputAdequacy.status, "blocked"); }); it("does not classify code-fence wording as a coding patch", () => { assert.equal(inferTaskType("Use emoji 😅 and code fence"), "general"); }); it("uses a JSON-only output contract for structured extraction", () => { const boosted = generateBoostedPrompt("Extrahiere Felder aus Notizen in JSON.", { taskType: "structured_extraction" }); const outputContract = boosted.sections.find((section) => section.label === "Output Contract")?.body || ""; assert.match(outputContract, /structured data only/); assert.match(outputContract, /no surrounding prose/); assert.doesNotMatch(outputContract, /Final deliverables/); assert.equal(boosted.inputAdequacy.status, "needs_clarification"); }); it("throws on unknown profile or harness ids", () => { assert.throws(() => findModelProfile("missing-profile"), /Unknown model adapter profile/); assert.throws(() => findHarnessPolicy("missing-harness"), /Unknown harness policy/); }); it("compares adapters deterministically for a prompt", () => { const comparison = compareAdapters("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt.", { taskType: "structured_extraction", harnessPolicyId: "blank" }); assert.equal(comparison.length, 10); assert.ok(comparison[0].score.overall >= comparison[comparison.length - 1].score.overall); assert.equal(comparison.every((item) => item.taskSpec.taskType === "structured_extraction"), true); }); it("labels local fixture comparison as heuristic prompt-profile comparison data", () => { const rows = runLocalFixtureComparison(); assert.equal(rows.length, 5); assert.equal(rows.every((row) => row.best.score.source === "prompt_shape_heuristic"), true); }); it("scores raw strings but prefers structured boosted prompts", () => { const weak = scorePrompt("Bitte mach das gut."); const boosted = generateBoostedPrompt("Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken."); assert.ok(boosted.score.overall > weak.overall); }); it("runs the eval harness through a pluggable runner", async () => { const runner: PromptRunner = { async run(prompt) { return { output: `Received prompt with ${prompt.length} characters. Verification complete.`, metadata: { model: "fake-runner", durationMs: 1 } }; } }; const result = await runEvalCase( { id: "TEST-01", taskType: "strategic_memo", rawUserPrompt: "Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken.", acceptanceCriteria: ["coherence", "risks"] }, runner, { modelProfileId: "claude" } ); assert.equal(result.caseId, "TEST-01"); assert.match(result.output, /Verification complete/); assert.equal(result.metadata?.model, "fake-runner"); assert.equal(result.scoreSource, "prompt_shape_heuristic"); assert.ok(result.score.overall > 0); }); it("uses custom judge scores and provenance when a JudgeRunner is provided", async () => { const runner: PromptRunner = { async run() { return { output: "model output", metadata: { runnerId: "fake-runner", harness: "blank" } }; } }; const judge: JudgeRunner = { async judge() { return { overall: 12, grade: "weak", source: "judge", dimensions: [], findings: ["custom judge score"] }; } }; const result = await runEvalCase( { id: "TEST-02", taskType: "general", rawUserPrompt: "Summarize the provided note.", acceptanceCriteria: ["summary"] }, runner, {}, judge ); assert.equal(result.scoreSource, "judge"); assert.equal(result.score.source, "judge"); assert.equal(result.metadata?.runnerId, "fake-runner"); }); });