File size: 6,178 Bytes
8ddfad6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 | name: document-classification-env
version: 1.0.0
description: >
A real-world document classification and routing environment where AI agents
learn to classify documents and route them to appropriate departments with
optimal speed and accuracy tradeoffs.
author: OpenEnv Community
license: MIT
# Environment specification
environment:
id: DocumentClassification-v1
entry_point: environment:DocumentClassificationEnv
# Observation space definition
observation_space:
type: dict
properties:
document_id:
type: string
description: Unique identifier for the document
content:
type: string
description: Raw text content of the document
word_count:
type: integer
description: Number of words in the document
min: 0
has_urgency_markers:
type: boolean
description: Whether document contains urgency indicators
features:
type: array
items:
type: number
description: Pre-extracted TF-IDF features (normalized)
shape: [100] # 100-dimensional feature vector
# Action space definition
action_space:
type: discrete
description: Category selection for document routing
# Values depend on difficulty level
# Easy: 5 categories, Medium: 10 categories, Hard: 20 categories
# Tasks definition
tasks:
- id: easy
name: Document Classification - Easy
difficulty: easy
description: >
Classify pre-processed documents into 5 categories with no time constraints.
Documents are pre-extracted and normalized. This task serves as an introduction
to the environment and API.
parameters:
num_documents: 100
num_categories: 5
categories:
- General
- Billing
- Support
- Technical
- HR
time_limit_per_action: null
time_limit_per_episode: null
grading:
type: accuracy_based
success_threshold: 0.95
test_dataset_size: 100
metrics:
- accuracy
- precision_per_category
- recall_per_category
- id: medium
name: Document Classification - Medium
difficulty: medium
description: >
Classify raw text documents into 10 categories with moderate complexity.
Includes time constraints per action. Requires the agent to balance
processing time with accuracy.
parameters:
num_documents: 500
num_categories: 10
categories:
- General
- Billing
- Billing-Dispute
- Support
- Technical
- Technical-Bug
- HR-Payroll
- HR-Benefits
- Legal
- Executive
time_limit_per_action: 2.0 # seconds
time_limit_per_episode: 1800 # 30 minutes for full dataset
grading:
type: accuracy_with_time_bonus
success_threshold: 0.85
test_dataset_size: 500
metrics:
- accuracy
- average_response_time
- speed_efficiency_ratio
- precision_per_category
- recall_per_category
- id: hard
name: Document Classification - Hard
difficulty: hard
description: >
Classify complex documents into 20 fine-grained categories with
challenging real-world scenarios. Tight time constraints create
speed-accuracy tradeoffs. Documents may contain mixed formats
and ambiguous classification cases.
parameters:
num_documents: 1000
num_categories: 20
categories:
- General
- Billing
- Billing-Dispute
- Billing-Refund
- Support-Urgent
- Support-Normal
- Technical
- Technical-Bug
- Technical-Feature
- HR-Payroll
- HR-Benefits
- HR-Complaint
- Legal
- Legal-Contract
- Legal-Compliance
- Executive
- Executive-Strategic
- Finance
- Marketing
- Operations
time_limit_per_action: 1.0 # seconds
time_limit_per_episode: 1200 # 20 minutes for full dataset
grading:
type: accuracy_with_speed_cost
success_threshold: 0.75
test_dataset_size: 1000
metrics:
- accuracy
- response_time
- speed_accuracy_tradeoff_score
- confusion_matrix
- f1_score_macro
- f1_score_weighted
# Reward function specification
reward:
type: composite
description: >
Composite reward combining accuracy, speed, and penalizing errors.
The exact calculation depends on task difficulty.
components:
- name: accuracy_bonus
description: Positive reward for correct classification
weight: 0.7
value_range: [0, 1]
- name: speed_bonus
description: Bonus reward for fast decision making
weight: 0.2
value_range: [0, 0.2]
scales_with: response_time_efficiency
- name: error_penalty
description: Penalty for incorrect classification
weight: 0.1
value_range: [-0.5, 0]
- name: processing_cost
description: Small deduction for resource usage (hard task only)
weight: 0.0
task_specific: hard
value_range: [-0.1, 0]
# Episode structure
episode:
max_steps: null # No step limit, episode ends when all documents processed
reset_regenerates_data: true
observation_includes_progress: true
done_conditions:
- all_documents_classified
- time_limit_exceeded
- manual_termination
# Baseline performance targets
baseline_targets:
easy:
score: 0.78
accuracy: 0.78
time_per_action_ms: 50
medium:
score: 0.65
accuracy: 0.68
time_per_action_ms: 150
speed_bonus: 0.15
hard:
score: 0.52
accuracy: 0.55
time_per_action_ms: 100
accuracy_std_dev: 0.05
# Reproducibility settings
reproducibility:
seed_controllable: true
deterministic_data_generation: true
deterministic_grading: true
fixed_test_sets: true
# Deployment settings
deployment:
containerized: true
docker_image: doc-classifier-env:latest
huggingface_spaces: true
api_endpoint: /api/step
inference_framework: gymnasium_compatible
|