| import type { |
| EvalCase, |
| EvalDimension, |
| HarnessPolicy, |
| ModelAdapterProfile, |
| TaskType |
| } from "./types.ts"; |
|
|
| export const taskTypeLabels: Record<TaskType, string> = { |
| long_document: "Langes Dokument", |
| coding_patch: "Coding Patch", |
| research_synthesis: "Research Synthese", |
| structured_extraction: "Strukturierte Extraktion", |
| strategic_memo: "Strategisches Memo", |
| general: "Allgemein" |
| }; |
|
|
| export const taskTypeHints: Record< |
| TaskType, |
| { |
| inputs: string[]; |
| deliverables: string[]; |
| constraints: string[]; |
| successCriteria: string[]; |
| } |
| > = { |
| long_document: { |
| inputs: ["source document or file attachment"], |
| deliverables: ["document map", "chunk plan", "source-grounded summary", "render or export script"], |
| constraints: ["use only source-grounded claims", "summarize locally before global synthesis"], |
| successCriteria: ["faithfulness", "coverage", "clear structure", "unsupported claims removed"] |
| }, |
| coding_patch: { |
| inputs: ["existing codebase", "bug report or failing behavior"], |
| deliverables: ["root-cause analysis", "minimal code patch", "focused tests", "verification summary"], |
| constraints: ["avoid unrelated refactors", "do not hardcode the fixture"], |
| successCriteria: ["bug fixed generally", "tests cover the regression", "existing behavior preserved"] |
| }, |
| research_synthesis: { |
| inputs: ["research question", "source material or current web research"], |
| deliverables: ["source list", "fact/inference split", "decision memo", "risks and next steps"], |
| constraints: ["prioritize primary sources", "date-sensitive claims require verification"], |
| successCriteria: ["current evidence", "citation discipline", "clear recommendation"] |
| }, |
| structured_extraction: { |
| inputs: ["unstructured notes or source text"], |
| deliverables: ["valid structured object", "field mapping notes", "null or uncertainty handling"], |
| constraints: ["return only the requested structure unless asked otherwise", "do not invent missing fields"], |
| successCriteria: ["valid syntax", "field accuracy", "stable missing-value policy"] |
| }, |
| strategic_memo: { |
| inputs: ["notes, goals, market or product context"], |
| deliverables: ["strategy memo", "tradeoffs", "risks", "next actions"], |
| constraints: ["avoid generic management filler", "state assumptions explicitly"], |
| successCriteria: ["coherent thesis", "concrete actions", "visible tradeoffs"] |
| }, |
| general: { |
| inputs: ["user-provided context"], |
| deliverables: ["completed requested output", "verification notes"], |
| constraints: ["preserve the user's task intent", "make assumptions explicit"], |
| successCriteria: ["complete answer", "clear constraints", "usable final artifact"] |
| } |
| }; |
|
|
| export const harnessPolicies: HarnessPolicy[] = [ |
| { |
| id: "blank", |
| label: "Blank Harness", |
| runtimeAssumptions: [ |
| "No specific agent framework, tool syntax, file system, or subagent feature is assumed.", |
| "The prompt must be usable in a generic runtime." |
| ], |
| toolRules: [ |
| "Mention tool use only as an optional capability when the task requires external inspection.", |
| "Do not require unavailable tools, hidden scratchpads, or provider-specific functions." |
| ], |
| artifactRules: [ |
| "Name deliverables clearly without assuming exact write paths.", |
| "Ask for machine-readable structure when the output is meant to be integrated." |
| ], |
| verificationRules: [ |
| "Require a visible verification pass before finalization.", |
| "Mark uncertainty instead of converting it into asserted facts." |
| ], |
| portabilityRules: [ |
| "Avoid model-specific chat-template tokens.", |
| "Keep the final prompt plain text and copyable." |
| ] |
| }, |
| { |
| id: "blank-coding", |
| label: "Blank Coding Harness", |
| runtimeAssumptions: [ |
| "The runtime can inspect local project files when available.", |
| "No subagent or repository layout is assumed." |
| ], |
| toolRules: [ |
| "Read before editing when the task touches code.", |
| "Run relevant checks when the environment makes that possible." |
| ], |
| artifactRules: [ |
| "Keep patches focused on the requested behavior.", |
| "Separate implementation summary from verification summary." |
| ], |
| verificationRules: [ |
| "Include regression tests or explain why they cannot be run.", |
| "Verify that unrelated behavior is not changed." |
| ], |
| portabilityRules: [ |
| "Do not depend on IDE-specific commands.", |
| "Avoid framework claims until the codebase confirms them." |
| ] |
| } |
| ]; |
|
|
| export const defaultRubric: EvalDimension[] = [ |
| { |
| key: "input_adequacy", |
| label: "Input Adequacy", |
| description: "The raw user prompt has enough task signal to justify a boosted draft.", |
| weight: 1.4 |
| }, |
| { |
| key: "task_preservation", |
| label: "Task Preservation", |
| description: "The boosted prompt keeps the observed user objective intact.", |
| weight: 1.2 |
| }, |
| { |
| key: "deliverable_completeness", |
| label: "Deliverables", |
| description: "Observed and adapter-suggested outputs are visibly separated.", |
| weight: 1 |
| }, |
| { |
| key: "format_clarity", |
| label: "Format Clarity", |
| description: "The expected output shape is clear enough to execute.", |
| weight: 0.9 |
| }, |
| { |
| key: "grounding", |
| label: "Grounding", |
| description: "The prompt contains surface signals for unsupported-claim control.", |
| weight: 1.1 |
| }, |
| { |
| key: "verification", |
| label: "Verification", |
| description: "The prompt contains visible verification instructions.", |
| weight: 1.1 |
| }, |
| { |
| key: "harness_portability", |
| label: "Harness Portability", |
| description: "The prompt appears free of unavailable runtime assumptions.", |
| weight: 1 |
| }, |
| { |
| key: "bloat_control", |
| label: "Bloat Control", |
| description: "The prompt adds useful structure without burying the task.", |
| weight: 0.7 |
| } |
| ]; |
|
|
| export const modelAdapterProfiles: ModelAdapterProfile[] = [ |
| { |
| id: "gpt-openai", |
| label: "GPT / OpenAI", |
| family: "gpt", |
| evidence: "strong_prior", |
| evidenceNote: "Official prompting/model documentation provides a useful prior; real strength still requires task-specific eval runs.", |
| summary: "Outcome-first, verification-first, concise structured sections.", |
| style: "Outcome-first, Verification-first", |
| explicitness: 70, |
| examples: 45, |
| verification: 90, |
| harnessRules: 60, |
| adapterRules: [ |
| "State the target outcome and quality gates directly.", |
| "Use a visible verification loop before final output.", |
| "Adapter hypothesis: prefer concise structured sections over heavy micromanagement." |
| ], |
| failureModes: ["missing verification", "overlong process text", "implicit acceptance criteria"], |
| promptPattern: "Task, Inputs, Deliverables, Quality Gates, Verification" |
| }, |
| { |
| id: "claude", |
| label: "Claude", |
| family: "claude", |
| evidence: "strong_prior", |
| evidenceNote: "Official prompting and long-context guidance supports this prior; runtime behavior must still be evaluated.", |
| summary: "Principle-first, XML-friendly, strong local-before-global discipline.", |
| style: "Principle-first, XML-friendly", |
| explicitness: 75, |
| examples: 55, |
| verification: 85, |
| harnessRules: 55, |
| adapterRules: [ |
| "Separate context, method, rules, and outputs.", |
| "Do local extraction before global synthesis.", |
| "Use positive epistemic rules: preserve uncertainty, avoid smoothing." |
| ], |
| failureModes: [ |
| "turns ambiguous source evidence into polished assertions", |
| "creates global synthesis before local extraction", |
| "leaves uncertainty unstated" |
| ], |
| promptPattern: "Context, Method, Rules, Outputs" |
| }, |
| { |
| id: "gemini", |
| label: "Gemini", |
| family: "gemini", |
| evidence: "strong_prior", |
| evidenceNote: "Official prompting strategy docs support this prior; no local eval evidence is bundled.", |
| summary: "Context-rich, response-format-led, benefits from examples.", |
| style: "Context-rich, Pattern-led", |
| explicitness: 82, |
| examples: 80, |
| verification: 70, |
| harnessRules: 45, |
| adapterRules: [ |
| "Make input, constraints, and response format visibly separate.", |
| "Include a small output pattern when format matters.", |
| "Treat provided context as the primary evidence base for grounded tasks." |
| ], |
| failureModes: ["format drift", "underspecified constraints", "weak missing-value policy"], |
| promptPattern: "Role, Input, Task, Constraints, Process, Output Format" |
| }, |
| { |
| id: "kimi", |
| label: "Kimi", |
| family: "kimi", |
| evidence: "moderate_prior", |
| evidenceNote: "Vendor docs support long-context and delimiter priors; profile details are still adapter hypotheses until evaluated.", |
| summary: "Long-context, delimiter-friendly, recursive chunk-merge workflow.", |
| style: "Long-context, Tool-aware", |
| explicitness: 78, |
| examples: 65, |
| verification: 76, |
| harnessRules: 80, |
| adapterRules: [ |
| "Use delimiters to separate source boundary from task.", |
| "Describe long-document workflows recursively: chunk, summarize, merge.", |
| "Clarify tool compatibility when the runtime offers tools." |
| ], |
| failureModes: ["source boundary leakage", "tool/runtime mismatch", "large-context drift"], |
| promptPattern: "System Role, Source Boundary, Task, Steps, Failure Policy" |
| }, |
| { |
| id: "glm", |
| label: "GLM", |
| family: "glm", |
| evidence: "moderate_prior", |
| evidenceNote: "Capability and prompting docs provide a working prior; generated behavior is not locally verified.", |
| summary: "System-role heavy, delimiter-based, thinking-aware.", |
| style: "System-role, Thinking-mode aware", |
| explicitness: 86, |
| examples: 58, |
| verification: 82, |
| harnessRules: 68, |
| adapterRules: [ |
| "Define role, style, and task mode up front.", |
| "Use delimiters for reference material.", |
| "Stabilize output as a list of artifacts." |
| ], |
| failureModes: ["role ambiguity", "loose artifact format", "verification omitted"], |
| promptPattern: "System, User Block, Work Method, Deliverables" |
| }, |
| { |
| id: "qwen", |
| label: "Qwen", |
| family: "qwen", |
| evidence: "moderate_prior", |
| evidenceNote: "Prompting and coder docs provide a working prior; exact adapter fit requires empirical runs.", |
| summary: "Example-stabilized, delimiter-friendly, explicit format rules.", |
| style: "Example-stabilized, Tool-compatible", |
| explicitness: 84, |
| examples: 78, |
| verification: 72, |
| harnessRules: 72, |
| adapterRules: [ |
| "Replace vague words with measurable requirements.", |
| "Include a mini output example when schema or style matters.", |
| "Specify tool steps only when tools are actually available." |
| ], |
| failureModes: ["vague target words", "schema drift", "unavailable tool assumptions"], |
| promptPattern: "Background, Objective, Input Delimiter, Required Output Example, Steps" |
| }, |
| { |
| id: "deepseek", |
| label: "DeepSeek", |
| family: "deepseek", |
| evidence: "moderate_prior", |
| evidenceNote: "Mode and structured-output docs support a moderate prior; broader prompting behavior is inferred.", |
| summary: "Mode-explicit, format-guarded, final-artifact separation.", |
| style: "Mode-explicit, Format-guarded", |
| explicitness: 88, |
| examples: 70, |
| verification: 78, |
| harnessRules: 64, |
| adapterRules: [ |
| "If the target provider exposes model modes, choose chat model or reasoner model outside this generic prompt.", |
| "For structured output, name the format and include the shape.", |
| "Adapter hypothesis: keep intermediate work separate from final artifacts." |
| ], |
| failureModes: ["mode mismatch", "JSON invalidity", "analysis leaks into final"], |
| promptPattern: "Mode, Goal, Important Rules, Required Artifacts" |
| }, |
| { |
| id: "mistral", |
| label: "Mistral", |
| family: "mistral", |
| evidence: "moderate_prior", |
| evidenceNote: "Prompting docs support structured prompting priors; profile strength is not yet measured locally.", |
| summary: "Structured, purpose-led, avoids vague instructions.", |
| style: "Structured, Purpose-led", |
| explicitness: 86, |
| examples: 82, |
| verification: 68, |
| harnessRules: 45, |
| adapterRules: [ |
| "Use clear purpose and user-task separation in the prompt text.", |
| "Replace vague quality adjectives with testable criteria.", |
| "Only request the artifacts needed for the user's goal." |
| ], |
| failureModes: ["ambiguous purpose", "blurry words", "unnecessary output"], |
| promptPattern: "System Purpose, Task, Required Sections, Quality Requirements" |
| }, |
| { |
| id: "minimax", |
| label: "MiniMax", |
| family: "minimax", |
| evidence: "hypothesis", |
| evidenceNote: "Concrete prompting evidence is limited in this project; all rules are failure-prevention hypotheses.", |
| summary: "Harness-first, guardrail-heavy, strict locality and anti-hallucination.", |
| style: "Harness-first, Guardrail-heavy", |
| explicitness: 87, |
| examples: 55, |
| verification: 86, |
| harnessRules: 84, |
| adapterRules: [ |
| "Adapter hypothesis: separate roles, local work, checks, and merge rights.", |
| "Adapter hypothesis: use hard anti-hallucination rules.", |
| "Adapter hypothesis: prevent local-to-global leakage before review." |
| ], |
| failureModes: [ |
| "merges unsupported claims into fluent synthesis", |
| "creates global synthesis before local extraction", |
| "mixes role responsibilities" |
| ], |
| promptPattern: "Goal, Roles, Hard Rules, Required Outputs, Parallel Execution" |
| }, |
| { |
| id: "llama", |
| label: "Llama", |
| family: "llama", |
| evidence: "moderate_prior", |
| evidenceNote: "Prompt-ops and chat-template docs support a moderate prior; open-weight variants require separate evals.", |
| summary: "Template-precise, example-heavy, eval-optimized.", |
| style: "Template-precise, Eval-optimized", |
| explicitness: 90, |
| examples: 88, |
| verification: 74, |
| harnessRules: 70, |
| adapterRules: [ |
| "Apply provider-specific chat templates outside this generic prompt when the runtime requires them.", |
| "Use output examples when format drift appears.", |
| "Adapter hypothesis: evaluate open-weight variants separately instead of assuming one profile." |
| ], |
| failureModes: ["template mismatch", "format drift", "variant-specific behavior"], |
| promptPattern: "System, User, Output Pattern, Example, Source Boundary" |
| } |
| ]; |
|
|
| export const evalFixtures: EvalCase[] = [ |
| { |
| id: "PB-01", |
| taskType: "long_document", |
| rawUserPrompt: |
| "Fasse ein langes PDF quellentreu in Markdown zusammen und erstelle ein PDF-Render-Skript.", |
| acceptanceCriteria: ["faithfulness", "coverage", "chronology", "render script"] |
| }, |
| { |
| id: "PB-02", |
| taskType: "coding_patch", |
| rawUserPrompt: "Finde den Bug in einem Modul, implementiere den Fix und liefere Tests.", |
| acceptanceCriteria: ["general fix", "tests", "minimal change", "no hardcoding"] |
| }, |
| { |
| id: "PB-03", |
| taskType: "research_synthesis", |
| rawUserPrompt: |
| "Recherchiere den aktuellen Stand eines technischen Themas und schreibe eine Entscheidungsvorlage.", |
| acceptanceCriteria: ["source quality", "recency", "fact/inference split", "recommendation"] |
| }, |
| { |
| id: "PB-04", |
| taskType: "structured_extraction", |
| rawUserPrompt: "Extrahiere Felder aus chaotischen Notizen in ein valides JSON-Objekt.", |
| acceptanceCriteria: ["valid JSON", "field accuracy", "null handling", "no extra prose"] |
| }, |
| { |
| id: "PB-05", |
| taskType: "strategic_memo", |
| rawUserPrompt: |
| "Erstelle aus Stichpunkten eine knappe Produktstrategie mit Risiken und naechsten Schritten.", |
| acceptanceCriteria: ["coherence", "assumptions", "tradeoffs", "concrete actions"] |
| } |
| ]; |
|
|
