Spaces:
Sleeping
Sleeping
File size: 1,951 Bytes
116524e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """Data contracts for the ACE pipeline — samples, environments, and step results."""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Dict, Optional
from .outputs import AgentOutput
@dataclass
class Sample:
"""Single task instance presented to ACE."""
question: str
context: str = ""
ground_truth: Optional[str] = None
metadata: Dict[str, object] = field(default_factory=dict)
id: Optional[str] = None
@dataclass
class EnvironmentResult:
"""Feedback returned by the task environment after evaluating agent output."""
feedback: str
ground_truth: Optional[str]
metrics: Dict[str, float] = field(default_factory=dict)
class TaskEnvironment(ABC):
"""Abstract interface for evaluating agent outputs."""
@abstractmethod
def evaluate(self, sample: Sample, agent_output: AgentOutput) -> EnvironmentResult:
"""Evaluate the agent's output for a given sample."""
class SimpleEnvironment(TaskEnvironment):
"""Built-in environment that checks if ground truth appears in the answer."""
def evaluate(self, sample: Sample, agent_output: AgentOutput) -> EnvironmentResult:
if not sample.ground_truth:
return EnvironmentResult(
feedback="No ground truth provided",
ground_truth=None,
metrics={"correct": 0.0},
)
answer = agent_output.final_answer.lower()
truth = sample.ground_truth.lower()
is_correct = truth in answer
return EnvironmentResult(
feedback=(
"Correct!"
if is_correct
else f"Incorrect. Expected: {sample.ground_truth}"
),
ground_truth=sample.ground_truth,
metrics={"correct": 1.0 if is_correct else 0.0},
)
|