import os import time import numpy as np import torch import torch.nn as nn import torch.optim as optim from environment import DocumentClassificationEnv class SimplePolicy(nn.Module): def __init__(self, input_dim, output_dim): super(SimplePolicy, self).__init__() self.fc = nn.Sequential( nn.Linear(input_dim, 256), nn.ReLU(), nn.Linear(256, output_dim), nn.Softmax(dim=-1) ) def forward(self, x): return self.fc(x) def train(difficulty="easy", num_episodes=50): print(f"--- Starting RL Training on {difficulty} ---") env = DocumentClassificationEnv(task_difficulty=difficulty) input_dim = env.feature_dim output_dim = env.num_categories # Simplify: ignore tools for now policy = SimplePolicy(input_dim, output_dim) optimizer = optim.Adam(policy.parameters(), lr=0.01) history = [] for ep in range(num_episodes): obs, _ = env.reset() done = False ep_reward = 0 while not done: # Convert features to tensor x = torch.FloatTensor(obs["features"]).unsqueeze(0) # Action probability distribution probs = policy(x) m = torch.distributions.Categorical(probs) action_tensor = m.sample() action = action_tensor.item() # Step in environment new_obs, reward, done, _, info = env.step(action) # Simple REINFORCE-like update (simplified for demo) # Log prob of action * reward loss = -m.log_prob(action_tensor) * reward optimizer.zero_grad() loss.backward() optimizer.step() obs = new_obs ep_reward += reward acc = env.correct / env.total if env.total > 0 else 0 history.append({"episode": ep+1, "reward": ep_reward, "accuracy": acc}) if (ep + 1) % 5 == 0: print(f"Episode {ep+1}/{num_episodes} | Reward: {ep_reward:.2f} | Accuracy: {acc:.2%}") print("--- Training Complete ---") return history if __name__ == "__main__": import argparse parser = argparse.ArgumentParser() parser.add_argument("--difficulty", type=str, default="easy") parser.add_argument("--episodes", type=int, default=20) args = parser.parse_args() train(difficulty=args.difficulty, num_episodes=args.episodes)