File size: 7,181 Bytes
8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 a2d30b7 8ddfad6 a2d30b7 8ddfad6 a2d30b7 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 49234f8 8ddfad6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 | name: document-classification-env
version: 1.0.0
description: >
A real-world document classification and routing environment where AI agents
learn to classify documents and route them to appropriate departments with
optimal speed and accuracy tradeoffs.
author: OpenEnv Community
license: MIT
# Environment specification
environment:
id: DocumentClassification-v1
entry_point: server.document_classification_environment:DocumentClassificationEnvironment
# Observation space definition
observation_space:
type: dict
properties:
document_id:
type: string
description: Unique identifier for the document
content:
type: string
description: Raw text content of the document
word_count:
type: integer
description: Number of words in the document
min: 0
has_urgency_markers:
type: boolean
description: Whether document contains urgency indicators
features:
type: array
items:
type: number
description: Pre-extracted TF-IDF features (normalized)
shape: [100] # 100-dimensional feature vector
metadata_response:
type: string
description: Output from the Request Metadata tool
sla_remaining:
type: number
description: Seconds remaining before SLA breach for current document
min: 0
# Action space definition
action_space:
type: discrete
description: Category selection or Tool Usage. Actions 0 to N-1 classify the document. Action N runs 'Request Metadata' tool. Action N+1 runs 'Escalate to Human'.
# Values depend on difficulty level
# Easy: 7 actions, Medium: 12 actions, Hard: 24 actions
# Tasks definition
tasks:
- id: easy
name: Document Classification - Easy
difficulty: easy
description: >
Classify pre-processed documents into 5 categories with no time constraints.
Documents are pre-extracted and normalized. This task serves as an introduction
to the environment and API.
grader: ".grading:AgentGrader"
parameters:
num_documents: 100
num_categories: 5
categories:
- General
- Billing
- Support
- Technical
- HR
time_limit_per_action: null
time_limit_per_episode: null
grading:
type: accuracy_based
success_threshold: 0.95
test_dataset_size: 100
metrics:
- accuracy
- precision_per_category
- recall_per_category
- id: medium
name: Document Classification - Medium
difficulty: medium
description: >
Classify raw text documents into 10 categories with moderate complexity.
Includes time constraints per action. Requires the agent to balance
processing time with accuracy.
grader: ".grading:AgentGrader"
parameters:
num_documents: 500
num_categories: 10
categories:
- General
- Billing
- Billing-Dispute
- Support
- Technical
- Technical-Bug
- HR-Payroll
- HR-Benefits
- Legal
- Executive
time_limit_per_action: 2.0 # seconds
time_limit_per_episode: 1800 # 30 minutes for full dataset
grading:
type: accuracy_with_time_bonus
success_threshold: 0.85
test_dataset_size: 500
metrics:
- accuracy
- average_response_time
- speed_efficiency_ratio
- precision_per_category
- recall_per_category
- id: hard
name: Document Classification - Hard
difficulty: hard
description: >
Classify complex documents into 20 fine-grained categories with
challenging real-world scenarios. Tight time constraints create
speed-accuracy tradeoffs. Documents may contain mixed formats
and ambiguous classification cases.
grader: ".grading:AgentGrader"
parameters:
num_documents: 1000
num_categories: 22
categories:
- General
- Billing
- Billing-Dispute
- Billing-Refund
- Support-Urgent
- Support-Normal
- Technical
- Technical-Bug
- Technical-Feature
- HR-Payroll
- HR-Benefits
- HR-Complaint
- Legal
- Legal-Contract
- Legal-Compliance
- Executive
- Executive-Strategic
- Finance
- Marketing
- Operations
- Finance-Tax
- Marketing-Social
time_limit_per_action: 1.0 # seconds
time_limit_per_episode: 1200 # 20 minutes for full dataset
grading:
type: accuracy_with_speed_cost
success_threshold: 0.75
test_dataset_size: 1000
metrics:
- accuracy
- response_time
- speed_accuracy_tradeoff_score
- confusion_matrix
- f1_score_macro
- f1_score_weighted
# Reward function specification
reward:
type: composite
description: >
Composite reward combining accuracy, speed, and penalizing errors.
The exact calculation depends on task difficulty.
components:
- name: accuracy_bonus
description: Positive reward for correct classification
weight: 0.7
value_range: [0, 1]
- name: speed_bonus
description: Bonus reward for fast decision making
weight: 0.2
value_range: [0, 0.2]
scales_with: response_time_efficiency
- name: error_penalty
description: Penalty for incorrect classification
weight: 0.1
value_range: [-0.5, 0]
- name: processing_cost
description: Small deduction (-0.01) for resource usage per action (hard task only)
weight: 0.1
task_specific: hard
value_range: [-0.01, 0]
- name: tool_usage_cost
description: Deduct -0.05 reward when agent requests metadata to penalize unnecessary tool usage.
weight: 0.05
value_range: [-0.05, 0]
- name: sla_penalty
description: Time-pressure penalty that scales with how long the agent deliberates on a document beyond the SLA window. Caps at -0.3.
weight: 0.1
value_range: [-0.3, 0]
task_specific: medium, hard
# Episode structure
episode:
max_steps: null # No step limit, episode ends when all documents processed
reset_regenerates_data: true
observation_includes_progress: true
done_conditions:
- all_documents_classified
- time_limit_exceeded
- manual_termination
# Baseline performance targets
baseline_targets:
easy:
score: 0.80
accuracy: 0.80
time_per_action_ms: 50
medium:
score: 0.79
accuracy: 0.79
time_per_action_ms: 150
speed_bonus: 0.15
hard:
score: 0.23
accuracy: 0.23
time_per_action_ms: 100
accuracy_std_dev: 0.05
# Reproducibility settings
reproducibility:
seed_controllable: true
deterministic_data_generation: true
deterministic_grading: true
fixed_test_sets: true
# Deployment settings
deployment:
containerized: true
docker_image: doc-classifier-env:latest
huggingface_spaces: true
api_endpoint: /api/step
inference_framework: gymnasium_compatible
|