| import assert from "node:assert/strict"; |
| import { describe, it } from "node:test"; |
| import { |
| analyzeUserPrompt, |
| compareAdapters, |
| findHarnessPolicy, |
| findModelProfile, |
| generateBoostedPrompt, |
| inferTaskType, |
| modelAdapterProfiles, |
| runEvalCase, |
| runLocalFixtureComparison, |
| scorePrompt |
| } from "../src/index.ts"; |
| import type { JudgeRunner, PromptRunner } from "../src/index.ts"; |
|
|
| describe("PromptBoost core", () => { |
| it("infers task types from raw user prompts", () => { |
| assert.equal( |
| inferTaskType("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt."), |
| "structured_extraction" |
| ); |
| assert.equal( |
| inferTaskType("Finde den Bug in einem Modul, implementiere den Fix und liefere Tests."), |
| "coding_patch" |
| ); |
| }); |
|
|
| it("creates a stable task spec without dropping the user intent", () => { |
| const raw = |
| "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript."; |
| const spec = analyzeUserPrompt(raw, { taskType: "long_document" }); |
|
|
| assert.equal(spec.rawUserPrompt, raw); |
| assert.equal(spec.taskType, "long_document"); |
| assert.match(spec.goal, /PDF/); |
| assert.ok(spec.deliverables.some((item) => item.includes("Markdown"))); |
| assert.ok(spec.suggestedScaffolding.deliverables.some((item) => item.includes("document map"))); |
| assert.ok(spec.constraints.some((item) => item.includes("Preserve the user's task intent"))); |
| }); |
|
|
| it("builds a harness-neutral boosted prompt with score and eval plan", () => { |
| const boosted = generateBoostedPrompt( |
| "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.", |
| { |
| taskType: "long_document", |
| modelProfileId: "gpt-openai", |
| harnessPolicyId: "blank" |
| } |
| ); |
|
|
| assert.match(boosted.prompt, /Task Spec/); |
| assert.match(boosted.prompt, /Adapter-Suggested Scaffolding/); |
| assert.match(boosted.prompt, /Harness Policy/); |
| assert.match(boosted.prompt, /Model Adapter/); |
| assert.match(boosted.prompt, /Verification/); |
| assert.match(boosted.prompt, /provider-neutral/); |
| assert.equal(boosted.evalPlan.metadataFields.includes("model"), true); |
| assert.ok(boosted.score.overall >= 75); |
| assert.equal(boosted.score.source, "prompt_shape_heuristic"); |
| }); |
|
|
| it("keeps all ten model adapter profiles available", () => { |
| assert.equal(modelAdapterProfiles.length, 10); |
| assert.deepEqual( |
| modelAdapterProfiles.map((profile) => profile.id), |
| [ |
| "gpt-openai", |
| "claude", |
| "gemini", |
| "kimi", |
| "glm", |
| "qwen", |
| "deepseek", |
| "mistral", |
| "minimax", |
| "llama" |
| ] |
| ); |
| assert.equal(modelAdapterProfiles.every((profile) => Boolean(profile.evidenceNote)), true); |
| assert.equal(modelAdapterProfiles.every((profile) => profile.evidence !== "strong"), true); |
| }); |
|
|
| it("blocks empty prompts instead of scoring the generated scaffold as strong", () => { |
| const boosted = generateBoostedPrompt(""); |
|
|
| assert.equal(boosted.inputAdequacy.status, "blocked"); |
| assert.equal(boosted.score.grade, "blocked"); |
| assert.ok(boosted.score.overall <= 35); |
| assert.equal(boosted.taskSpec.deliverables.length, 0); |
| assert.ok(boosted.taskSpec.suggestedScaffolding.deliverables.length > 0); |
| }); |
|
|
| it("keeps very short summarize prompts generic instead of inventing long-document work", () => { |
| const boosted = generateBoostedPrompt("Summarize."); |
|
|
| assert.equal(boosted.taskSpec.taskType, "general"); |
| assert.equal(boosted.taskSpec.deliverables.includes("document map"), false); |
| assert.equal(boosted.inputAdequacy.status, "needs_clarification"); |
| assert.ok(boosted.score.overall < 62); |
| }); |
|
|
| it("preserves non-ASCII terminal punctuation in goals", () => { |
| const boosted = generateBoostedPrompt("请用中文总结这段内容。"); |
|
|
| assert.equal(boosted.taskSpec.goal, "请用中文总结这段内容。"); |
| assert.notEqual(boosted.inputAdequacy.status, "blocked"); |
| }); |
|
|
| it("does not classify code-fence wording as a coding patch", () => { |
| assert.equal(inferTaskType("Use emoji 😅 and code fence"), "general"); |
| }); |
|
|
| it("uses a JSON-only output contract for structured extraction", () => { |
| const boosted = generateBoostedPrompt("Extrahiere Felder aus Notizen in JSON.", { |
| taskType: "structured_extraction" |
| }); |
| const outputContract = boosted.sections.find((section) => section.label === "Output Contract")?.body || ""; |
|
|
| assert.match(outputContract, /structured data only/); |
| assert.match(outputContract, /no surrounding prose/); |
| assert.doesNotMatch(outputContract, /Final deliverables/); |
| assert.equal(boosted.inputAdequacy.status, "needs_clarification"); |
| }); |
|
|
| it("throws on unknown profile or harness ids", () => { |
| assert.throws(() => findModelProfile("missing-profile"), /Unknown model adapter profile/); |
| assert.throws(() => findHarnessPolicy("missing-harness"), /Unknown harness policy/); |
| }); |
|
|
| it("compares adapters deterministically for a prompt", () => { |
| const comparison = compareAdapters("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt.", { |
| taskType: "structured_extraction", |
| harnessPolicyId: "blank" |
| }); |
|
|
| assert.equal(comparison.length, 10); |
| assert.ok(comparison[0].score.overall >= comparison[comparison.length - 1].score.overall); |
| assert.equal(comparison.every((item) => item.taskSpec.taskType === "structured_extraction"), true); |
| }); |
|
|
| it("labels local fixture comparison as heuristic prompt-profile comparison data", () => { |
| const rows = runLocalFixtureComparison(); |
|
|
| assert.equal(rows.length, 5); |
| assert.equal(rows.every((row) => row.best.score.source === "prompt_shape_heuristic"), true); |
| }); |
|
|
| it("scores raw strings but prefers structured boosted prompts", () => { |
| const weak = scorePrompt("Bitte mach das gut."); |
| const boosted = generateBoostedPrompt("Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken."); |
|
|
| assert.ok(boosted.score.overall > weak.overall); |
| }); |
|
|
| it("runs the eval harness through a pluggable runner", async () => { |
| const runner: PromptRunner = { |
| async run(prompt) { |
| return { |
| output: `Received prompt with ${prompt.length} characters. Verification complete.`, |
| metadata: { |
| model: "fake-runner", |
| durationMs: 1 |
| } |
| }; |
| } |
| }; |
|
|
| const result = await runEvalCase( |
| { |
| id: "TEST-01", |
| taskType: "strategic_memo", |
| rawUserPrompt: "Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken.", |
| acceptanceCriteria: ["coherence", "risks"] |
| }, |
| runner, |
| { modelProfileId: "claude" } |
| ); |
|
|
| assert.equal(result.caseId, "TEST-01"); |
| assert.match(result.output, /Verification complete/); |
| assert.equal(result.metadata?.model, "fake-runner"); |
| assert.equal(result.scoreSource, "prompt_shape_heuristic"); |
| assert.ok(result.score.overall > 0); |
| }); |
|
|
| it("uses custom judge scores and provenance when a JudgeRunner is provided", async () => { |
| const runner: PromptRunner = { |
| async run() { |
| return { |
| output: "model output", |
| metadata: { |
| runnerId: "fake-runner", |
| harness: "blank" |
| } |
| }; |
| } |
| }; |
| const judge: JudgeRunner = { |
| async judge() { |
| return { |
| overall: 12, |
| grade: "weak", |
| source: "judge", |
| dimensions: [], |
| findings: ["custom judge score"] |
| }; |
| } |
| }; |
|
|
| const result = await runEvalCase( |
| { |
| id: "TEST-02", |
| taskType: "general", |
| rawUserPrompt: "Summarize the provided note.", |
| acceptanceCriteria: ["summary"] |
| }, |
| runner, |
| {}, |
| judge |
| ); |
|
|
| assert.equal(result.scoreSource, "judge"); |
| assert.equal(result.score.source, "judge"); |
| assert.equal(result.metadata?.runnerId, "fake-runner"); |
| }); |
| }); |
|
|
