import gymnasium as gym from gymnasium import spaces import numpy as np import time import random from tasks import TaskDataGenerator, get_task_config from constants import CATEGORIES class DocumentClassificationEnv(gym.Env): metadata = { "render_modes": ["human"], "name": "DocumentClassification-v1" } # Different difficulty levels = different number of categories # Easy = 5, Medium = 10, Hard = 22 (gets gnarly fast) # Category Maps are now centralized in constants.py CATEGORY_MAPS = CATEGORIES def __init__(self, task_difficulty="easy", seed=None): super().__init__() self.task_difficulty = task_difficulty if task_difficulty not in ["easy", "medium", "hard"]: raise ValueError(f"Pick easy, medium, or hard. Got: {task_difficulty}") if seed is not None: np.random.seed(seed) random.seed(seed) self.task_config = get_task_config(task_difficulty) self.num_categories = self.task_config["num_categories"] self.feature_dim = self.task_config["feature_dim"] self.action_space = spaces.Discrete(self.num_categories + 2) # +2 for Tools self.observation_space = spaces.Dict({ "document_id": spaces.Text(max_length=20), "content": spaces.Text(max_length=10000), "word_count": spaces.Box(low=0, high=10000, shape=(1,), dtype=np.int32), "has_urgency_markers": spaces.MultiBinary(1), "features": spaces.Box(low=-1.0, high=1.0, shape=(self.feature_dim,), dtype=np.float32), "document_index": spaces.Box(low=0, high=10000, shape=(1,), dtype=np.int32), "total_documents": spaces.Box(low=0, high=10000, shape=(1,), dtype=np.int32), "metadata_response": spaces.Text(max_length=1000), "sla_remaining": spaces.Box(low=0, high=300, shape=(1,), dtype=np.float32), }) self.data_generator = TaskDataGenerator(task_difficulty, seed) self.sla_seconds = self.task_config.get("sla_seconds", None) self.documents = None self.labels = None self.current_document_index = 0 self.episode_rewards = [] self.correct = 0 self.total = 0 self.episode_times = [] self.current_metadata_response = "Not requested. Use action to request customer history." self.doc_start_time = None # Tracks time spent on current document self.sla_breaches = 0 # Count of SLA breaches self.start_time = None self.action_history = [] self.reward_history = [] self.processing_times = [] def reset(self, seed=None, options=None): if seed is not None: np.random.seed(seed) random.seed(seed) self.documents, self.labels = self.data_generator.generate_task_data() self.current_document_index = 0 self.episode_rewards = [] self.correct = 0 self.total = 0 self.episode_times = [] self.action_history = [] self.reward_history = [] self.processing_times = [] self.current_metadata_response = "Not requested. Use action to request customer history." self.doc_start_time = time.time() self.sla_breaches = 0 self.start_time = time.time() return self._obs(), {} def _compute_sla_penalty(self): """Compute SLA penalty based on time spent on current document. The longer the agent deliberates, the higher the penalty.""" if self.sla_seconds is None or self.doc_start_time is None: return 0.0 elapsed = time.time() - self.doc_start_time if elapsed > self.sla_seconds: self.sla_breaches += 1 # Penalty scales with how much over SLA (capped at -0.3) overage_ratio = min((elapsed - self.sla_seconds) / self.sla_seconds, 1.0) return -0.3 * overage_ratio return 0.0 def step(self, action): t0 = time.time() # Enforce per-episode time limits limit_per_ep = self.task_config.get("time_limit_per_episode") if limit_per_ep is not None and self.start_time is not None: if time.time() - self.start_time > limit_per_ep: info = {"episode_complete": True, "time_limit_exceeded": True, "episode_accuracy": self.correct / self.total if self.total > 0 else 0.0} info["episode_summary"] = self._summary() return self._obs(), 0.0, True, False, info if self.current_document_index >= len(self.documents): return self._obs(), 0.0, True, False, {"episode_complete": True} true_label = self.labels[self.current_document_index] true_cat_str = self.CATEGORY_MAPS[self.task_difficulty][true_label] sla_penalty = self._compute_sla_penalty() # Tool: Request Metadata if action == self.num_categories: if random.random() < 0.15: self.current_metadata_response = "[SYSTEM: Database timeout. No history available.]" else: self.current_metadata_response = f"[SYSTEM: User history suggests high correlation with {true_cat_str.split('-')[0]}]" tool_reward = -0.05 + sla_penalty # Tool cost + SLA pressure self.episode_rewards.append(tool_reward) self.episode_times.append(time.time() - t0) info = { "is_correct": False, "true_category": true_cat_str, "predicted_category": "TOOL_METADATA", "processing_time_ms": (time.time() - t0) * 1000, "episode_accuracy": self.correct / self.total if self.total > 0 else 0.0, "sla_penalty": sla_penalty, } return self._obs(), tool_reward, False, False, info # Tool: Escalate to Human elif action == self.num_categories + 1: self.current_metadata_response = "Not requested. Use action to request customer history." esc_reward = 0.0 + sla_penalty # Escalation + SLA pressure self.episode_rewards.append(esc_reward) self.total += 1 self.episode_times.append(time.time() - t0) self.current_document_index += 1 self.doc_start_time = time.time() # Reset SLA timer for next doc done = self.current_document_index >= len(self.documents) info = { "is_correct": False, "true_category": true_cat_str, "predicted_category": "ESCALATED", "processing_time_ms": (time.time() - t0) * 1000, "episode_accuracy": self.correct / self.total if self.total > 0 else 0.0, "sla_penalty": sla_penalty, } if done: info["episode_summary"] = self._summary() return self._obs(), esc_reward, done, False, info # Standard Classification self.current_metadata_response = "Not requested. Use action to request customer history." is_correct = action == true_label processing_time = time.time() - t0 is_vip = self.documents[self.current_document_index].get("is_vip", False) error_penalty = -0.5 if is_vip else -0.4 base_reward = 1.0 if is_correct else error_penalty # Speed Bonus: Scale reward if classification is correct and fast speed_bonus = 0.0 if is_correct and self.task_difficulty in ["medium", "hard"]: # Bonus if under 1.0s (medium) or 0.5s (hard) threshold = 1.0 if self.task_difficulty == "medium" else 0.5 if processing_time < threshold: speed_bonus = 0.2 * (1.0 - (processing_time / threshold)) # Processing Cost: Fixed deduction for resource usage (Hard only) proc_cost = -0.01 if self.task_difficulty == "hard" else 0.0 reward = base_reward + sla_penalty + speed_bonus + proc_cost self.episode_rewards.append(reward) self.action_history.append(action) self.reward_history.append(reward) self.processing_times.append(processing_time) if is_correct: self.correct += 1 self.total += 1 self.episode_times.append(processing_time) self.current_document_index += 1 self.doc_start_time = time.time() # Reset SLA timer for next doc done = self.current_document_index >= len(self.documents) obs = self._obs() try: pred_cat = self.CATEGORY_MAPS[self.task_difficulty][action] except KeyError: pred_cat = f"INVALID_ACTION_{action}" info = { "is_correct": is_correct, "true_category": true_cat_str, "predicted_category": pred_cat, "processing_time_ms": processing_time * 1000, "episode_accuracy": self.correct / self.total if self.total > 0 else 0.0, "sla_penalty": sla_penalty, "speed_bonus": speed_bonus, "proc_cost": proc_cost, } if done: info["episode_summary"] = self._summary() return obs, reward, done, False, info def state(self): """Return the full internal environment state with rich diagnostic history.""" # Ground-truth label for the current document (hidden from agent) current_true_label = None current_true_category = None if self.documents and self.current_document_index < len(self.documents): true_label_idx = self.labels[self.current_document_index] current_true_label = int(true_label_idx) current_true_category = self.CATEGORY_MAPS[self.task_difficulty].get(current_true_label) return { # ── Progress ────────────────────────────────────────────────── "document_index": self.current_document_index, "total_documents": len(self.documents) if self.documents else 0, # ── Hidden ground truth (not in observation) ────────────────── "current_true_label": current_true_label, "current_true_category": current_true_category, # ── Accumulated metrics ─────────────────────────────────────── "total_reward": sum(self.episode_rewards), "correct": self.correct, "total_classified": self.total, "accuracy": self.correct / self.total if self.total > 0 else 0.0, "avg_time_ms": float(np.mean(self.episode_times) * 1000) if self.episode_times else 0.0, "sla_breaches": self.sla_breaches, # ── History Traces (NEW) ────────────────────────────────────── "action_history": list(self.action_history), "reward_history": [float(r) for r in self.reward_history], "processing_times_ms": [float(t * 1000) for t in self.processing_times], # ── Environment config ──────────────────────────────────────── "difficulty": self.task_difficulty, "num_categories": self.num_categories, "sla_seconds": self.sla_seconds, } def _get_sla_remaining(self): """Calculate SLA time remaining for current document.""" if self.sla_seconds is None or self.doc_start_time is None: return 999.0 # No SLA pressure elapsed = time.time() - self.doc_start_time return max(0.0, self.sla_seconds - elapsed) def _obs(self): sla_remaining = self._get_sla_remaining() if self.current_document_index >= len(self.documents): return { "document_id": "", "content": "", "word_count": np.array([0], dtype=np.int32), "has_urgency_markers": np.array([0], dtype=np.int8), "features": np.zeros(self.feature_dim, dtype=np.float32), "document_index": np.array([self.current_document_index], dtype=np.int32), "total_documents": np.array([len(self.documents)], dtype=np.int32), "metadata_response": self.current_metadata_response, "sla_remaining": np.array([0.0], dtype=np.float32), } doc = self.documents[self.current_document_index] return { "document_id": doc["id"], "content": doc["content"], "word_count": np.array([doc["word_count"]], dtype=np.int32), "has_urgency_markers": np.array([int(doc["has_urgency_markers"])], dtype=np.int8), "features": np.array(doc["features"], dtype=np.float32), "document_index": np.array([self.current_document_index], dtype=np.int32), "total_documents": np.array([len(self.documents)], dtype=np.int32), "metadata_response": self.current_metadata_response, "sla_remaining": np.array([sla_remaining], dtype=np.float32), } def _summary(self): total_reward = sum(self.episode_rewards) avg_reward = total_reward / len(self.episode_rewards) if self.episode_rewards else 0.0 acc = self.correct / self.total if self.total > 0 else 0.0 avg_time = np.mean(self.episode_times) * 1000 if self.episode_times else 0.0 elapsed = time.time() - self.start_time return { "total_reward": total_reward, "avg_reward": avg_reward, "accuracy": acc, "total_classified": self.total, "avg_time_ms": avg_time, "total_time_s": elapsed, } def render(self): pass def close(self): pass