File size: 2,526 Bytes
1b64cdd
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
import os
import time
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from environment import DocumentClassificationEnv

class SimplePolicy(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(SimplePolicy, self).__init__()
        self.fc = nn.Sequential(
            nn.Linear(input_dim, 256),
            nn.ReLU(),
            nn.Linear(256, output_dim),
            nn.Softmax(dim=-1)
        )

    def forward(self, x):
        return self.fc(x)

def train(difficulty="easy", num_episodes=50):
    print(f"--- Starting RL Training on {difficulty} ---")
    env = DocumentClassificationEnv(task_difficulty=difficulty)
    input_dim = env.feature_dim
    output_dim = env.num_categories # Simplify: ignore tools for now
    
    policy = SimplePolicy(input_dim, output_dim)
    optimizer = optim.Adam(policy.parameters(), lr=0.01)
    
    history = []
    
    for ep in range(num_episodes):
        obs, _ = env.reset()
        done = False
        ep_reward = 0
        
        while not done:
            # Convert features to tensor
            x = torch.FloatTensor(obs["features"]).unsqueeze(0)
            
            # Action probability distribution
            probs = policy(x)
            m = torch.distributions.Categorical(probs)
            action_tensor = m.sample()
            action = action_tensor.item()
            
            # Step in environment
            new_obs, reward, done, _, info = env.step(action)
            
            # Simple REINFORCE-like update (simplified for demo)
            # Log prob of action * reward
            loss = -m.log_prob(action_tensor) * reward
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            
            obs = new_obs
            ep_reward += reward
            
        acc = env.correct / env.total if env.total > 0 else 0
        history.append({"episode": ep+1, "reward": ep_reward, "accuracy": acc})
        
        if (ep + 1) % 5 == 0:
            print(f"Episode {ep+1}/{num_episodes} | Reward: {ep_reward:.2f} | Accuracy: {acc:.2%}")
            
    print("--- Training Complete ---")
    return history

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("--difficulty", type=str, default="easy")
    parser.add_argument("--episodes", type=int, default=20)
    args = parser.parse_args()
    
    train(difficulty=args.difficulty, num_episodes=args.episodes)