| export interface EvaluationScoreSummary { |
| source: string | null |
| score: number | null |
| valid: boolean | null |
| failureReason: string | null |
| method: string | null |
| verdict: string | null |
| targetAcquired: boolean | null |
| confidence: number | null |
| needsHumanReview: boolean |
| geometry: number | null |
| dimensions: number | null |
| annotations: number | null |
| presentation: number | null |
| } |
|
|
| export interface RunSummary { |
| id: string |
| taskId: string |
| attemptId: string | null |
| modelId: string |
| provider: string |
| reasoningEffort: string | null |
| status: string |
| stopReason: string | null |
| failure: string | null |
| acceptedActions: number |
| invalidActions: number |
| screenshots: number |
| elapsedSeconds: number | null |
| artifactBytes: number |
| score: number | null |
| valid: boolean | null |
| scoreFailure: string | null |
| scoreMethod: string | null |
| scoreSource: string | null |
| verdict: string | null |
| targetAcquired: boolean | null |
| judgeConfidence: number | null |
| needsHumanReview: boolean |
| visionScore: EvaluationScoreSummary | null |
| deterministicScore: EvaluationScoreSummary | null |
| evaluationState: string |
| evaluationStage: string | null |
| evaluationError: string | null |
| updatedAt: string |
| } |
|
|
| export interface FrameArtifact { |
| name: string |
| sha256: string | null |
| url: string |
| } |
|
|
| export interface ModelTurn { |
| turn: number |
| tool_use_id?: string |
| action: Record<string, unknown> |
| stop_reason: string |
| usage: Record<string, number> |
| latency_ms: number |
| text: string[] |
| discarded_tool_uses: number |
| model_input?: Record<string, unknown> | null |
| } |
|
|
| export interface InputReceipt { |
| action_index: number |
| request_id?: string | null |
| idempotent_replay?: boolean |
| receipt: Record<string, unknown> | null |
| action?: Record<string, unknown> | null |
| tool_use_id?: string | null |
| batch_index?: number |
| batch_size?: number |
| batch_start_action_index?: number |
| } |
|
|
| export interface TraceDetail { |
| summary: RunSummary |
| state: Record<string, unknown> | null |
| result: Record<string, unknown> | null |
| score: Record<string, unknown> | null |
| scores: { |
| vision: Record<string, unknown> | null |
| deterministic: Record<string, unknown> | null |
| } |
| turns: ModelTurn[] |
| receipts: InputReceipt[] |
| traceIndexed: boolean |
| traceNotice: string | null |
| frames: FrameArtifact[] |
| referenceUrl: string |
| artifacts: { |
| dwg: string | null |
| candidateRender: string | null |
| diffRender: string | null |
| } |
| } |
|
|
| export interface BenchmarkUsage { |
| inputTokens: number |
| cachedInputTokens: number |
| outputTokens: number |
| reasoningOutputTokens: number |
| totalTokens: number |
| modelTurns: number |
| apiCost: number | null |
| } |
|
|
| export interface BenchmarkLevelSummary { |
| total: number |
| completed: number |
| scored: number |
| meanScore: number | null |
| cost: number |
| } |
|
|
| export interface BenchmarkDrawingTypeSummary { |
| total: number |
| scored: number |
| meanScore: number | null |
| } |
|
|
| export interface BenchmarkAggregate { |
| totalTasks: number |
| queuedTasks: number |
| activeTasks: number |
| settledTasks: number |
| completedTasks: number |
| recoveredTasks: number |
| budgetExhaustedTasks: number |
| failedTasks: number |
| infrastructureFailedTasks: number |
| artifactMissingTasks: number |
| artifactTasks: number |
| scoredTasks: number |
| meanScore: number | null |
| visionScoredTasks: number |
| visionMeanScore: number | null |
| deterministicScoredTasks: number |
| deterministicMeanScore: number | null |
| acceptedActions: number |
| invalidActions: number |
| aggregateElapsedSeconds: number |
| usage: BenchmarkUsage |
| levels: Record<string, BenchmarkLevelSummary> |
| drawingTypes: Record<string, BenchmarkDrawingTypeSummary> |
| evaluationStates: Record<string, number> |
| executionStates: Record<string, number> |
| verdicts: Record<string, number> |
| } |
|
|
| export interface BenchmarkSummary { |
| id: string |
| batchId: string | null |
| name: string |
| status: string |
| modelId: string |
| provider: string |
| models: string[] |
| providers: string[] |
| startedAt: string | null |
| updatedAt: string | null |
| maxConcurrency: number | null |
| aggregate: BenchmarkAggregate |
| } |
|
|
| export interface BenchmarkTask { |
| rolloutId: string |
| runId: string |
| taskId: string |
| level: string |
| drawingType: string |
| split: string |
| displayName: string |
| modelId: string |
| provider: string |
| status: string |
| stopReason: string | null |
| failure: string | null |
| acceptedActions: number |
| invalidActions: number |
| elapsedSeconds: number | null |
| setupElapsedSeconds: number | null |
| rolloutElapsedSeconds: number | null |
| artifactBytes: number |
| score: number | null |
| valid: boolean | null |
| scoreFailure: string | null |
| scoreMethod: string | null |
| scoreSource: string | null |
| verdict: string | null |
| targetAcquired: boolean | null |
| judgeConfidence: number | null |
| needsHumanReview: boolean |
| visionScore: EvaluationScoreSummary | null |
| deterministicScore: EvaluationScoreSummary | null |
| geometrySimilarity: number | null |
| dimensionSimilarity: number | null |
| annotationSimilarity: number | null |
| visualSimilarity: number | null |
| evaluationState: string |
| evaluationStage: string | null |
| evaluationError: string | null |
| startedAt: string | null |
| finishedAt: string | null |
| updatedAt: string | null |
| usage: BenchmarkUsage |
| } |
|
|
| export interface BenchmarkDetail extends BenchmarkSummary { |
| finishedAt: string | null |
| pricing: { |
| currency?: string |
| input_per_million: number |
| cached_input_per_million: number |
| output_per_million: number |
| source_url?: string | null |
| effective_date?: string | null |
| } | null |
| tasks: BenchmarkTask[] |
| } |
|
|
