TanujInsane
fix: add grader field to all 3 tasks in openenv.yaml
a2d30b7
Raw History Blame Contribute Delete
7.18 kB
name: document-classification-env
version: 1.0.0
description: >
A real-world document classification and routing environment where AI agents
learn to classify documents and route them to appropriate departments with
optimal speed and accuracy tradeoffs.
author: OpenEnv Community
license: MIT
# Environment specification
environment:
id: DocumentClassification-v1
entry_point: server.document_classification_environment:DocumentClassificationEnvironment
# Observation space definition
observation_space:
type: dict
properties:
document_id:
type: string
description: Unique identifier for the document
content:
type: string
description: Raw text content of the document
word_count:
type: integer
description: Number of words in the document
min: 0
has_urgency_markers:
type: boolean
description: Whether document contains urgency indicators
features:
type: array
items:
type: number
description: Pre-extracted TF-IDF features (normalized)
shape: [100] # 100-dimensional feature vector
metadata_response:
type: string
description: Output from the Request Metadata tool
sla_remaining:
type: number
description: Seconds remaining before SLA breach for current document
min: 0
# Action space definition
action_space:
type: discrete
description: Category selection or Tool Usage. Actions 0 to N-1 classify the document. Action N runs 'Request Metadata' tool. Action N+1 runs 'Escalate to Human'.
# Values depend on difficulty level
# Easy: 7 actions, Medium: 12 actions, Hard: 24 actions
# Tasks definition
tasks:
- id: easy
name: Document Classification - Easy
difficulty: easy
description: >
Classify pre-processed documents into 5 categories with no time constraints.
Documents are pre-extracted and normalized. This task serves as an introduction
to the environment and API.
grader: ".grading:AgentGrader"
parameters:
num_documents: 100
num_categories: 5
categories:
- General
- Billing
- Support
- Technical
- HR
time_limit_per_action: null
time_limit_per_episode: null
grading:
type: accuracy_based
success_threshold: 0.95
test_dataset_size: 100
metrics:
- accuracy
- precision_per_category
- recall_per_category
- id: medium
name: Document Classification - Medium
difficulty: medium
description: >
Classify raw text documents into 10 categories with moderate complexity.
Includes time constraints per action. Requires the agent to balance
processing time with accuracy.
grader: ".grading:AgentGrader"
parameters:
num_documents: 500
num_categories: 10
categories:
- General
- Billing
- Billing-Dispute
- Support
- Technical
- Technical-Bug
- HR-Payroll
- HR-Benefits
- Legal
- Executive
time_limit_per_action: 2.0 # seconds
time_limit_per_episode: 1800 # 30 minutes for full dataset
grading:
type: accuracy_with_time_bonus
success_threshold: 0.85
test_dataset_size: 500
metrics:
- accuracy
- average_response_time
- speed_efficiency_ratio
- precision_per_category
- recall_per_category
- id: hard
name: Document Classification - Hard
difficulty: hard
description: >
Classify complex documents into 20 fine-grained categories with
challenging real-world scenarios. Tight time constraints create
speed-accuracy tradeoffs. Documents may contain mixed formats
and ambiguous classification cases.
grader: ".grading:AgentGrader"
parameters:
num_documents: 1000
num_categories: 22
categories:
- General
- Billing
- Billing-Dispute
- Billing-Refund
- Support-Urgent
- Support-Normal
- Technical
- Technical-Bug
- Technical-Feature
- HR-Payroll
- HR-Benefits
- HR-Complaint
- Legal
- Legal-Contract
- Legal-Compliance
- Executive
- Executive-Strategic
- Finance
- Marketing
- Operations
- Finance-Tax
- Marketing-Social
time_limit_per_action: 1.0 # seconds
time_limit_per_episode: 1200 # 20 minutes for full dataset
grading:
type: accuracy_with_speed_cost
success_threshold: 0.75
test_dataset_size: 1000
metrics:
- accuracy
- response_time
- speed_accuracy_tradeoff_score
- confusion_matrix
- f1_score_macro
- f1_score_weighted
# Reward function specification
reward:
type: composite
description: >
Composite reward combining accuracy, speed, and penalizing errors.
The exact calculation depends on task difficulty.
components:
- name: accuracy_bonus
description: Positive reward for correct classification
weight: 0.7
value_range: [0, 1]
- name: speed_bonus
description: Bonus reward for fast decision making
weight: 0.2
value_range: [0, 0.2]
scales_with: response_time_efficiency
- name: error_penalty
description: Penalty for incorrect classification
weight: 0.1
value_range: [-0.5, 0]
- name: processing_cost
description: Small deduction (-0.01) for resource usage per action (hard task only)
weight: 0.1
task_specific: hard
value_range: [-0.01, 0]
- name: tool_usage_cost
description: Deduct -0.05 reward when agent requests metadata to penalize unnecessary tool usage.
weight: 0.05
value_range: [-0.05, 0]
- name: sla_penalty
description: Time-pressure penalty that scales with how long the agent deliberates on a document beyond the SLA window. Caps at -0.3.
weight: 0.1
value_range: [-0.3, 0]
task_specific: medium, hard
# Episode structure
episode:
max_steps: null # No step limit, episode ends when all documents processed
reset_regenerates_data: true
observation_includes_progress: true
done_conditions:
- all_documents_classified
- time_limit_exceeded
- manual_termination
# Baseline performance targets
baseline_targets:
easy:
score: 0.80
accuracy: 0.80
time_per_action_ms: 50
medium:
score: 0.79
accuracy: 0.79
time_per_action_ms: 150
speed_bonus: 0.15
hard:
score: 0.23
accuracy: 0.23
time_per_action_ms: 100
accuracy_std_dev: 0.05
# Reproducibility settings
reproducibility:
seed_controllable: true
deterministic_data_generation: true
deterministic_grading: true
fixed_test_sets: true
# Deployment settings
deployment:
containerized: true
docker_image: doc-classifier-env:latest
huggingface_spaces: true
api_endpoint: /api/step
inference_framework: gymnasium_compatible