File size: 2,526 Bytes
1b64cdd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | import os
import time
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from environment import DocumentClassificationEnv
class SimplePolicy(nn.Module):
def __init__(self, input_dim, output_dim):
super(SimplePolicy, self).__init__()
self.fc = nn.Sequential(
nn.Linear(input_dim, 256),
nn.ReLU(),
nn.Linear(256, output_dim),
nn.Softmax(dim=-1)
)
def forward(self, x):
return self.fc(x)
def train(difficulty="easy", num_episodes=50):
print(f"--- Starting RL Training on {difficulty} ---")
env = DocumentClassificationEnv(task_difficulty=difficulty)
input_dim = env.feature_dim
output_dim = env.num_categories # Simplify: ignore tools for now
policy = SimplePolicy(input_dim, output_dim)
optimizer = optim.Adam(policy.parameters(), lr=0.01)
history = []
for ep in range(num_episodes):
obs, _ = env.reset()
done = False
ep_reward = 0
while not done:
# Convert features to tensor
x = torch.FloatTensor(obs["features"]).unsqueeze(0)
# Action probability distribution
probs = policy(x)
m = torch.distributions.Categorical(probs)
action_tensor = m.sample()
action = action_tensor.item()
# Step in environment
new_obs, reward, done, _, info = env.step(action)
# Simple REINFORCE-like update (simplified for demo)
# Log prob of action * reward
loss = -m.log_prob(action_tensor) * reward
optimizer.zero_grad()
loss.backward()
optimizer.step()
obs = new_obs
ep_reward += reward
acc = env.correct / env.total if env.total > 0 else 0
history.append({"episode": ep+1, "reward": ep_reward, "accuracy": acc})
if (ep + 1) % 5 == 0:
print(f"Episode {ep+1}/{num_episodes} | Reward: {ep_reward:.2f} | Accuracy: {acc:.2%}")
print("--- Training Complete ---")
return history
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--difficulty", type=str, default="easy")
parser.add_argument("--episodes", type=int, default=20)
args = parser.parse_args()
train(difficulty=args.difficulty, num_episodes=args.episodes)
|