tests/core.test.ts

import assert from "node:assert/strict";
import { describe, it } from "node:test";
import {
  analyzeUserPrompt,
  compareAdapters,
  findHarnessPolicy,
  findModelProfile,
  generateBoostedPrompt,
  inferTaskType,
  modelAdapterProfiles,
  runEvalCase,
  runLocalFixtureComparison,
  scorePrompt
} from "../src/index.ts";
import type { JudgeRunner, PromptRunner } from "../src/index.ts";

describe("PromptBoost core", () => {
  it("infers task types from raw user prompts", () => {
    assert.equal(
      inferTaskType("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt."),
      "structured_extraction"
    );
    assert.equal(
      inferTaskType("Finde den Bug in einem Modul, implementiere den Fix und liefere Tests."),
      "coding_patch"
    );
  });

  it("creates a stable task spec without dropping the user intent", () => {
    const raw =
      "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.";
    const spec = analyzeUserPrompt(raw, { taskType: "long_document" });

    assert.equal(spec.rawUserPrompt, raw);
    assert.equal(spec.taskType, "long_document");
    assert.match(spec.goal, /PDF/);
    assert.ok(spec.deliverables.some((item) => item.includes("Markdown")));
    assert.ok(spec.suggestedScaffolding.deliverables.some((item) => item.includes("document map")));
    assert.ok(spec.constraints.some((item) => item.includes("Preserve the user's task intent")));
  });

  it("builds a harness-neutral boosted prompt with score and eval plan", () => {
    const boosted = generateBoostedPrompt(
      "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.",
      {
        taskType: "long_document",
        modelProfileId: "gpt-openai",
        harnessPolicyId: "blank"
      }
    );

    assert.match(boosted.prompt, /Task Spec/);
    assert.match(boosted.prompt, /Adapter-Suggested Scaffolding/);
    assert.match(boosted.prompt, /Harness Policy/);
    assert.match(boosted.prompt, /Model Adapter/);
    assert.match(boosted.prompt, /Verification/);
    assert.match(boosted.prompt, /provider-neutral/);
    assert.equal(boosted.evalPlan.metadataFields.includes("model"), true);
    assert.ok(boosted.score.overall >= 75);
    assert.equal(boosted.score.source, "prompt_shape_heuristic");
  });

  it("keeps all ten model adapter profiles available", () => {
    assert.equal(modelAdapterProfiles.length, 10);
    assert.deepEqual(
      modelAdapterProfiles.map((profile) => profile.id),
      [
        "gpt-openai",
        "claude",
        "gemini",
        "kimi",
        "glm",
        "qwen",
        "deepseek",
        "mistral",
        "minimax",
        "llama"
      ]
    );
    assert.equal(modelAdapterProfiles.every((profile) => Boolean(profile.evidenceNote)), true);
    assert.equal(modelAdapterProfiles.every((profile) => profile.evidence !== "strong"), true);
  });

  it("blocks empty prompts instead of scoring the generated scaffold as strong", () => {
    const boosted = generateBoostedPrompt("");

    assert.equal(boosted.inputAdequacy.status, "blocked");
    assert.equal(boosted.score.grade, "blocked");
    assert.ok(boosted.score.overall <= 35);
    assert.equal(boosted.taskSpec.deliverables.length, 0);
    assert.ok(boosted.taskSpec.suggestedScaffolding.deliverables.length > 0);
  });

  it("keeps very short summarize prompts generic instead of inventing long-document work", () => {
    const boosted = generateBoostedPrompt("Summarize.");

    assert.equal(boosted.taskSpec.taskType, "general");
    assert.equal(boosted.taskSpec.deliverables.includes("document map"), false);
    assert.equal(boosted.inputAdequacy.status, "needs_clarification");
    assert.ok(boosted.score.overall < 62);
  });

  it("preserves non-ASCII terminal punctuation in goals", () => {
    const boosted = generateBoostedPrompt("请用中文总结这段内容。");

    assert.equal(boosted.taskSpec.goal, "请用中文总结这段内容。");
    assert.notEqual(boosted.inputAdequacy.status, "blocked");
  });

  it("does not classify code-fence wording as a coding patch", () => {
    assert.equal(inferTaskType("Use emoji 😅 and code fence"), "general");
  });

  it("uses a JSON-only output contract for structured extraction", () => {
    const boosted = generateBoostedPrompt("Extrahiere Felder aus Notizen in JSON.", {
      taskType: "structured_extraction"
    });
    const outputContract = boosted.sections.find((section) => section.label === "Output Contract")?.body || "";

    assert.match(outputContract, /structured data only/);
    assert.match(outputContract, /no surrounding prose/);
    assert.doesNotMatch(outputContract, /Final deliverables/);
    assert.equal(boosted.inputAdequacy.status, "needs_clarification");
  });

  it("throws on unknown profile or harness ids", () => {
    assert.throws(() => findModelProfile("missing-profile"), /Unknown model adapter profile/);
    assert.throws(() => findHarnessPolicy("missing-harness"), /Unknown harness policy/);
  });

  it("compares adapters deterministically for a prompt", () => {
    const comparison = compareAdapters("Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt.", {
      taskType: "structured_extraction",
      harnessPolicyId: "blank"
    });

    assert.equal(comparison.length, 10);
    assert.ok(comparison[0].score.overall >= comparison[comparison.length - 1].score.overall);
    assert.equal(comparison.every((item) => item.taskSpec.taskType === "structured_extraction"), true);
  });

  it("labels local fixture comparison as heuristic prompt-profile comparison data", () => {
    const rows = runLocalFixtureComparison();

    assert.equal(rows.length, 5);
    assert.equal(rows.every((row) => row.best.score.source === "prompt_shape_heuristic"), true);
  });

  it("scores raw strings but prefers structured boosted prompts", () => {
    const weak = scorePrompt("Bitte mach das gut.");
    const boosted = generateBoostedPrompt("Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken.");

    assert.ok(boosted.score.overall > weak.overall);
  });

  it("runs the eval harness through a pluggable runner", async () => {
    const runner: PromptRunner = {
      async run(prompt) {
        return {
          output: `Received prompt with ${prompt.length} characters. Verification complete.`,
          metadata: {
            model: "fake-runner",
            durationMs: 1
          }
        };
      }
    };

    const result = await runEvalCase(
      {
        id: "TEST-01",
        taskType: "strategic_memo",
        rawUserPrompt: "Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken.",
        acceptanceCriteria: ["coherence", "risks"]
      },
      runner,
      { modelProfileId: "claude" }
    );

    assert.equal(result.caseId, "TEST-01");
    assert.match(result.output, /Verification complete/);
    assert.equal(result.metadata?.model, "fake-runner");
    assert.equal(result.scoreSource, "prompt_shape_heuristic");
    assert.ok(result.score.overall > 0);
  });

  it("uses custom judge scores and provenance when a JudgeRunner is provided", async () => {
    const runner: PromptRunner = {
      async run() {
        return {
          output: "model output",
          metadata: {
            runnerId: "fake-runner",
            harness: "blank"
          }
        };
      }
    };
    const judge: JudgeRunner = {
      async judge() {
        return {
          overall: 12,
          grade: "weak",
          source: "judge",
          dimensions: [],
          findings: ["custom judge score"]
        };
      }
    };

    const result = await runEvalCase(
      {
        id: "TEST-02",
        taskType: "general",
        rawUserPrompt: "Summarize the provided note.",
        acceptanceCriteria: ["summary"]
      },
      runner,
      {},
      judge
    );

    assert.equal(result.scoreSource, "judge");
    assert.equal(result.score.source, "judge");
    assert.equal(result.metadata?.runnerId, "fake-runner");
  });
});