| export type TaskType = |
| | "long_document" |
| | "coding_patch" |
| | "research_synthesis" |
| | "structured_extraction" |
| | "strategic_memo" |
| | "general"; |
|
|
| export type EvalDimensionKey = |
| | "input_adequacy" |
| | "task_preservation" |
| | "deliverable_completeness" |
| | "format_clarity" |
| | "grounding" |
| | "verification" |
| | "harness_portability" |
| | "bloat_control"; |
|
|
| export type EvidenceLevel = "strong_prior" | "moderate_prior" | "hypothesis"; |
|
|
| export type ScoreSource = "prompt_shape_heuristic" | "judge" | "unscored"; |
|
|
| export interface SuggestedScaffolding { |
| inputs: string[]; |
| deliverables: string[]; |
| constraints: string[]; |
| successCriteria: string[]; |
| } |
|
|
| export interface InputAdequacy { |
| status: "blocked" | "needs_clarification" | "sufficient"; |
| score: number; |
| reasons: string[]; |
| warnings: string[]; |
| } |
|
|
| export interface TaskSpec { |
| id: string; |
| rawUserPrompt: string; |
| taskType: TaskType; |
| goal: string; |
| inputs: string[]; |
| deliverables: string[]; |
| constraints: string[]; |
| successCriteria: string[]; |
| ambiguities: string[]; |
| forbiddenChanges: string[]; |
| suggestedScaffolding: SuggestedScaffolding; |
| inputAdequacy: InputAdequacy; |
| } |
|
|
| export interface HarnessPolicy { |
| id: string; |
| label: string; |
| runtimeAssumptions: string[]; |
| toolRules: string[]; |
| artifactRules: string[]; |
| verificationRules: string[]; |
| portabilityRules: string[]; |
| } |
|
|
| export interface ModelAdapterProfile { |
| id: string; |
| label: string; |
| family: string; |
| evidence: EvidenceLevel; |
| evidenceNote: string; |
| summary: string; |
| style: string; |
| explicitness: number; |
| examples: number; |
| verification: number; |
| harnessRules: number; |
| adapterRules: string[]; |
| failureModes: string[]; |
| promptPattern: string; |
| } |
|
|
| export interface PromptSection { |
| label: string; |
| body: string; |
| } |
|
|
| export interface BoostedPrompt { |
| id: string; |
| rawUserPrompt: string; |
| taskSpec: TaskSpec; |
| harnessPolicy: HarnessPolicy; |
| modelProfile: ModelAdapterProfile; |
| sections: PromptSection[]; |
| prompt: string; |
| evalPlan: EvalPlan; |
| rationale: string[]; |
| score: PromptScore; |
| inputAdequacy: InputAdequacy; |
| } |
|
|
| export interface EvalDimension { |
| key: EvalDimensionKey; |
| label: string; |
| description: string; |
| weight: number; |
| } |
|
|
| export interface EvalPlan { |
| rubric: EvalDimension[]; |
| expectedFailureModes: string[]; |
| requiredChecks: string[]; |
| metadataFields: string[]; |
| } |
|
|
| export interface PromptScoreDimension { |
| key: EvalDimensionKey; |
| label: string; |
| value: number; |
| evidence: string; |
| } |
|
|
| export interface PromptScore { |
| overall: number; |
| grade: "blocked" | "weak" | "usable" | "strong"; |
| source: ScoreSource; |
| dimensions: PromptScoreDimension[]; |
| findings: string[]; |
| } |
|
|
| export interface EvalCase { |
| id: string; |
| taskType: TaskType; |
| rawUserPrompt: string; |
| acceptanceCriteria: string[]; |
| context?: Record<string, unknown>; |
| artifacts?: string[]; |
| expectedOutputShape?: string; |
| } |
|
|
| export interface PromptRunResult { |
| output: string; |
| metadata?: { |
| runnerId?: string; |
| runnerVersion?: string; |
| harness?: string; |
| model?: string; |
| promptVersion?: string; |
| costUsd?: number; |
| durationMs?: number; |
| tokensIn?: number; |
| tokensOut?: number; |
| observedFailureModes?: string[]; |
| blockedChecks?: string[]; |
| extra?: Record<string, unknown>; |
| }; |
| } |
|
|
| export interface PromptRunner { |
| run(prompt: string, evalCase: EvalCase, boostedPrompt: BoostedPrompt): Promise<PromptRunResult>; |
| } |
|
|
| export interface JudgeRunner { |
| judge( |
| output: PromptRunResult, |
| evalCase: EvalCase, |
| boostedPrompt: BoostedPrompt |
| ): Promise<PromptScore>; |
| } |
|
|
| export interface EvalResult { |
| caseId: string; |
| boostedPromptId: string; |
| output: string; |
| score: PromptScore; |
| scoreSource: ScoreSource; |
| durationMs: number; |
| metadata: PromptRunResult["metadata"]; |
| judgeMetadata?: { |
| judgeId?: string; |
| judgeVersion?: string; |
| blockedChecks?: string[]; |
| observedFailureModes?: string[]; |
| extra?: Record<string, unknown>; |
| }; |
| } |
|
|
| export interface GenerateOptions { |
| taskType?: TaskType; |
| modelProfileId?: string; |
| harnessPolicyId?: string; |
| } |
|
|
