src/profiles.ts

import type {
  EvalCase,
  EvalDimension,
  HarnessPolicy,
  ModelAdapterProfile,
  TaskType
} from "./types.ts";

export const taskTypeLabels: Record<TaskType, string> = {
  long_document: "Langes Dokument",
  coding_patch: "Coding Patch",
  research_synthesis: "Research Synthese",
  structured_extraction: "Strukturierte Extraktion",
  strategic_memo: "Strategisches Memo",
  general: "Allgemein"
};

export const taskTypeHints: Record<
  TaskType,
  {
    inputs: string[];
    deliverables: string[];
    constraints: string[];
    successCriteria: string[];
  }
> = {
  long_document: {
    inputs: ["source document or file attachment"],
    deliverables: ["document map", "chunk plan", "source-grounded summary", "render or export script"],
    constraints: ["use only source-grounded claims", "summarize locally before global synthesis"],
    successCriteria: ["faithfulness", "coverage", "clear structure", "unsupported claims removed"]
  },
  coding_patch: {
    inputs: ["existing codebase", "bug report or failing behavior"],
    deliverables: ["root-cause analysis", "minimal code patch", "focused tests", "verification summary"],
    constraints: ["avoid unrelated refactors", "do not hardcode the fixture"],
    successCriteria: ["bug fixed generally", "tests cover the regression", "existing behavior preserved"]
  },
  research_synthesis: {
    inputs: ["research question", "source material or current web research"],
    deliverables: ["source list", "fact/inference split", "decision memo", "risks and next steps"],
    constraints: ["prioritize primary sources", "date-sensitive claims require verification"],
    successCriteria: ["current evidence", "citation discipline", "clear recommendation"]
  },
  structured_extraction: {
    inputs: ["unstructured notes or source text"],
    deliverables: ["valid structured object", "field mapping notes", "null or uncertainty handling"],
    constraints: ["return only the requested structure unless asked otherwise", "do not invent missing fields"],
    successCriteria: ["valid syntax", "field accuracy", "stable missing-value policy"]
  },
  strategic_memo: {
    inputs: ["notes, goals, market or product context"],
    deliverables: ["strategy memo", "tradeoffs", "risks", "next actions"],
    constraints: ["avoid generic management filler", "state assumptions explicitly"],
    successCriteria: ["coherent thesis", "concrete actions", "visible tradeoffs"]
  },
  general: {
    inputs: ["user-provided context"],
    deliverables: ["completed requested output", "verification notes"],
    constraints: ["preserve the user's task intent", "make assumptions explicit"],
    successCriteria: ["complete answer", "clear constraints", "usable final artifact"]
  }
};

export const harnessPolicies: HarnessPolicy[] = [
  {
    id: "blank",
    label: "Blank Harness",
    runtimeAssumptions: [
      "No specific agent framework, tool syntax, file system, or subagent feature is assumed.",
      "The prompt must be usable in a generic runtime."
    ],
    toolRules: [
      "Mention tool use only as an optional capability when the task requires external inspection.",
      "Do not require unavailable tools, hidden scratchpads, or provider-specific functions."
    ],
    artifactRules: [
      "Name deliverables clearly without assuming exact write paths.",
      "Ask for machine-readable structure when the output is meant to be integrated."
    ],
    verificationRules: [
      "Require a visible verification pass before finalization.",
      "Mark uncertainty instead of converting it into asserted facts."
    ],
    portabilityRules: [
      "Avoid model-specific chat-template tokens.",
      "Keep the final prompt plain text and copyable."
    ]
  },
  {
    id: "blank-coding",
    label: "Blank Coding Harness",
    runtimeAssumptions: [
      "The runtime can inspect local project files when available.",
      "No subagent or repository layout is assumed."
    ],
    toolRules: [
      "Read before editing when the task touches code.",
      "Run relevant checks when the environment makes that possible."
    ],
    artifactRules: [
      "Keep patches focused on the requested behavior.",
      "Separate implementation summary from verification summary."
    ],
    verificationRules: [
      "Include regression tests or explain why they cannot be run.",
      "Verify that unrelated behavior is not changed."
    ],
    portabilityRules: [
      "Do not depend on IDE-specific commands.",
      "Avoid framework claims until the codebase confirms them."
    ]
  }
];

export const defaultRubric: EvalDimension[] = [
  {
    key: "input_adequacy",
    label: "Input Adequacy",
    description: "The raw user prompt has enough task signal to justify a boosted draft.",
    weight: 1.4
  },
  {
    key: "task_preservation",
    label: "Task Preservation",
    description: "The boosted prompt keeps the observed user objective intact.",
    weight: 1.2
  },
  {
    key: "deliverable_completeness",
    label: "Deliverables",
    description: "Observed and adapter-suggested outputs are visibly separated.",
    weight: 1
  },
  {
    key: "format_clarity",
    label: "Format Clarity",
    description: "The expected output shape is clear enough to execute.",
    weight: 0.9
  },
  {
    key: "grounding",
    label: "Grounding",
    description: "The prompt contains surface signals for unsupported-claim control.",
    weight: 1.1
  },
  {
    key: "verification",
    label: "Verification",
    description: "The prompt contains visible verification instructions.",
    weight: 1.1
  },
  {
    key: "harness_portability",
    label: "Harness Portability",
    description: "The prompt appears free of unavailable runtime assumptions.",
    weight: 1
  },
  {
    key: "bloat_control",
    label: "Bloat Control",
    description: "The prompt adds useful structure without burying the task.",
    weight: 0.7
  }
];

export const modelAdapterProfiles: ModelAdapterProfile[] = [
  {
    id: "gpt-openai",
    label: "GPT / OpenAI",
    family: "gpt",
    evidence: "strong_prior",
    evidenceNote: "Official prompting/model documentation provides a useful prior; real strength still requires task-specific eval runs.",
    summary: "Outcome-first, verification-first, concise structured sections.",
    style: "Outcome-first, Verification-first",
    explicitness: 70,
    examples: 45,
    verification: 90,
    harnessRules: 60,
    adapterRules: [
      "State the target outcome and quality gates directly.",
      "Use a visible verification loop before final output.",
      "Adapter hypothesis: prefer concise structured sections over heavy micromanagement."
    ],
    failureModes: ["missing verification", "overlong process text", "implicit acceptance criteria"],
    promptPattern: "Task, Inputs, Deliverables, Quality Gates, Verification"
  },
  {
    id: "claude",
    label: "Claude",
    family: "claude",
    evidence: "strong_prior",
    evidenceNote: "Official prompting and long-context guidance supports this prior; runtime behavior must still be evaluated.",
    summary: "Principle-first, XML-friendly, strong local-before-global discipline.",
    style: "Principle-first, XML-friendly",
    explicitness: 75,
    examples: 55,
    verification: 85,
    harnessRules: 55,
    adapterRules: [
      "Separate context, method, rules, and outputs.",
      "Do local extraction before global synthesis.",
      "Use positive epistemic rules: preserve uncertainty, avoid smoothing."
    ],
    failureModes: [
      "turns ambiguous source evidence into polished assertions",
      "creates global synthesis before local extraction",
      "leaves uncertainty unstated"
    ],
    promptPattern: "Context, Method, Rules, Outputs"
  },
  {
    id: "gemini",
    label: "Gemini",
    family: "gemini",
    evidence: "strong_prior",
    evidenceNote: "Official prompting strategy docs support this prior; no local eval evidence is bundled.",
    summary: "Context-rich, response-format-led, benefits from examples.",
    style: "Context-rich, Pattern-led",
    explicitness: 82,
    examples: 80,
    verification: 70,
    harnessRules: 45,
    adapterRules: [
      "Make input, constraints, and response format visibly separate.",
      "Include a small output pattern when format matters.",
      "Treat provided context as the primary evidence base for grounded tasks."
    ],
    failureModes: ["format drift", "underspecified constraints", "weak missing-value policy"],
    promptPattern: "Role, Input, Task, Constraints, Process, Output Format"
  },
  {
    id: "kimi",
    label: "Kimi",
    family: "kimi",
    evidence: "moderate_prior",
    evidenceNote: "Vendor docs support long-context and delimiter priors; profile details are still adapter hypotheses until evaluated.",
    summary: "Long-context, delimiter-friendly, recursive chunk-merge workflow.",
    style: "Long-context, Tool-aware",
    explicitness: 78,
    examples: 65,
    verification: 76,
    harnessRules: 80,
    adapterRules: [
      "Use delimiters to separate source boundary from task.",
      "Describe long-document workflows recursively: chunk, summarize, merge.",
      "Clarify tool compatibility when the runtime offers tools."
    ],
    failureModes: ["source boundary leakage", "tool/runtime mismatch", "large-context drift"],
    promptPattern: "System Role, Source Boundary, Task, Steps, Failure Policy"
  },
  {
    id: "glm",
    label: "GLM",
    family: "glm",
    evidence: "moderate_prior",
    evidenceNote: "Capability and prompting docs provide a working prior; generated behavior is not locally verified.",
    summary: "System-role heavy, delimiter-based, thinking-aware.",
    style: "System-role, Thinking-mode aware",
    explicitness: 86,
    examples: 58,
    verification: 82,
    harnessRules: 68,
    adapterRules: [
      "Define role, style, and task mode up front.",
      "Use delimiters for reference material.",
      "Stabilize output as a list of artifacts."
    ],
    failureModes: ["role ambiguity", "loose artifact format", "verification omitted"],
    promptPattern: "System, User Block, Work Method, Deliverables"
  },
  {
    id: "qwen",
    label: "Qwen",
    family: "qwen",
    evidence: "moderate_prior",
    evidenceNote: "Prompting and coder docs provide a working prior; exact adapter fit requires empirical runs.",
    summary: "Example-stabilized, delimiter-friendly, explicit format rules.",
    style: "Example-stabilized, Tool-compatible",
    explicitness: 84,
    examples: 78,
    verification: 72,
    harnessRules: 72,
    adapterRules: [
      "Replace vague words with measurable requirements.",
      "Include a mini output example when schema or style matters.",
      "Specify tool steps only when tools are actually available."
    ],
    failureModes: ["vague target words", "schema drift", "unavailable tool assumptions"],
    promptPattern: "Background, Objective, Input Delimiter, Required Output Example, Steps"
  },
  {
    id: "deepseek",
    label: "DeepSeek",
    family: "deepseek",
    evidence: "moderate_prior",
    evidenceNote: "Mode and structured-output docs support a moderate prior; broader prompting behavior is inferred.",
    summary: "Mode-explicit, format-guarded, final-artifact separation.",
    style: "Mode-explicit, Format-guarded",
    explicitness: 88,
    examples: 70,
    verification: 78,
    harnessRules: 64,
    adapterRules: [
      "If the target provider exposes model modes, choose chat model or reasoner model outside this generic prompt.",
      "For structured output, name the format and include the shape.",
      "Adapter hypothesis: keep intermediate work separate from final artifacts."
    ],
    failureModes: ["mode mismatch", "JSON invalidity", "analysis leaks into final"],
    promptPattern: "Mode, Goal, Important Rules, Required Artifacts"
  },
  {
    id: "mistral",
    label: "Mistral",
    family: "mistral",
    evidence: "moderate_prior",
    evidenceNote: "Prompting docs support structured prompting priors; profile strength is not yet measured locally.",
    summary: "Structured, purpose-led, avoids vague instructions.",
    style: "Structured, Purpose-led",
    explicitness: 86,
    examples: 82,
    verification: 68,
    harnessRules: 45,
    adapterRules: [
      "Use clear purpose and user-task separation in the prompt text.",
      "Replace vague quality adjectives with testable criteria.",
      "Only request the artifacts needed for the user's goal."
    ],
    failureModes: ["ambiguous purpose", "blurry words", "unnecessary output"],
    promptPattern: "System Purpose, Task, Required Sections, Quality Requirements"
  },
  {
    id: "minimax",
    label: "MiniMax",
    family: "minimax",
    evidence: "hypothesis",
    evidenceNote: "Concrete prompting evidence is limited in this project; all rules are failure-prevention hypotheses.",
    summary: "Harness-first, guardrail-heavy, strict locality and anti-hallucination.",
    style: "Harness-first, Guardrail-heavy",
    explicitness: 87,
    examples: 55,
    verification: 86,
    harnessRules: 84,
    adapterRules: [
      "Adapter hypothesis: separate roles, local work, checks, and merge rights.",
      "Adapter hypothesis: use hard anti-hallucination rules.",
      "Adapter hypothesis: prevent local-to-global leakage before review."
    ],
    failureModes: [
      "merges unsupported claims into fluent synthesis",
      "creates global synthesis before local extraction",
      "mixes role responsibilities"
    ],
    promptPattern: "Goal, Roles, Hard Rules, Required Outputs, Parallel Execution"
  },
  {
    id: "llama",
    label: "Llama",
    family: "llama",
    evidence: "moderate_prior",
    evidenceNote: "Prompt-ops and chat-template docs support a moderate prior; open-weight variants require separate evals.",
    summary: "Template-precise, example-heavy, eval-optimized.",
    style: "Template-precise, Eval-optimized",
    explicitness: 90,
    examples: 88,
    verification: 74,
    harnessRules: 70,
    adapterRules: [
      "Apply provider-specific chat templates outside this generic prompt when the runtime requires them.",
      "Use output examples when format drift appears.",
      "Adapter hypothesis: evaluate open-weight variants separately instead of assuming one profile."
    ],
    failureModes: ["template mismatch", "format drift", "variant-specific behavior"],
    promptPattern: "System, User, Output Pattern, Example, Source Boundary"
  }
];

export const evalFixtures: EvalCase[] = [
  {
    id: "PB-01",
    taskType: "long_document",
    rawUserPrompt:
      "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.",
    acceptanceCriteria: ["faithfulness", "coverage", "chronology", "render script"]
  },
  {
    id: "PB-02",
    taskType: "coding_patch",
    rawUserPrompt: "Finde den Bug in einem Modul, implementiere den Fix und liefere Tests.",
    acceptanceCriteria: ["general fix", "tests", "minimal change", "no hardcoding"]
  },
  {
    id: "PB-03",
    taskType: "research_synthesis",
    rawUserPrompt:
      "Recherchiere den aktuellen Stand eines technischen Themas und schreibe eine Entscheidungsvorlage.",
    acceptanceCriteria: ["source quality", "recency", "fact/inference split", "recommendation"]
  },
  {
    id: "PB-04",
    taskType: "structured_extraction",
    rawUserPrompt: "Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt.",
    acceptanceCriteria: ["valid JSON", "field accuracy", "null handling", "no extra prose"]
  },
  {
    id: "PB-05",
    taskType: "strategic_memo",
    rawUserPrompt:
      "Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken und naechsten Schritten.",
    acceptanceCriteria: ["coherence", "assumptions", "tradeoffs", "concrete actions"]
  }
];