Delete train.py
Browse files
train.py
DELETED
|
@@ -1,240 +0,0 @@
|
|
| 1 |
-
# ==========================================
|
| 2 |
-
# 0. SIMPLE AUTHENTICATION (FOR GATED MODEL)
|
| 3 |
-
# ==========================================
|
| 4 |
-
import os
|
| 5 |
-
import sys
|
| 6 |
-
from huggingface_hub import login
|
| 7 |
-
|
| 8 |
-
# Grab the secret you set in the HF Space UI
|
| 9 |
-
hf_token = os.environ.get("HF_TOKEN")
|
| 10 |
-
|
| 11 |
-
if not hf_token:
|
| 12 |
-
print("❌ ERROR: HF_TOKEN not found in Secrets. Llama-3.2 download will fail.")
|
| 13 |
-
sys.exit(1)
|
| 14 |
-
|
| 15 |
-
# Silently log in so Unsloth can download the gated weights
|
| 16 |
-
login(token=hf_token.strip())
|
| 17 |
-
print("✅ Authenticated. Ready to load Llama-3.2.")
|
| 18 |
-
|
| 19 |
-
# ==========================================
|
| 20 |
-
# 1. THE STABLE ENVIRONMENT INSTALL
|
| 21 |
-
# ==========================================
|
| 22 |
-
# (Commented out for local runs, judges will run these if necessary)
|
| 23 |
-
import os
|
| 24 |
-
os.system('pip install "torch>=2.4.0,<2.11.0" torchvision torchaudio')
|
| 25 |
-
os.system('pip install unsloth unsloth_zoo')
|
| 26 |
-
os.system('pip install "trl==0.22.0" peft accelerate bitsandbytes openenv-core datasets matplotlib pandas')
|
| 27 |
-
|
| 28 |
-
import re
|
| 29 |
-
import json
|
| 30 |
-
import torch
|
| 31 |
-
import matplotlib.pyplot as plt
|
| 32 |
-
from tqdm import tqdm
|
| 33 |
-
from datasets import Dataset
|
| 34 |
-
from unsloth import FastLanguageModel
|
| 35 |
-
from trl import SFTTrainer
|
| 36 |
-
from transformers import TrainingArguments
|
| 37 |
-
|
| 38 |
-
# Import the environment we just ensured exists
|
| 39 |
-
from env import EnterpriseEmailEnv
|
| 40 |
-
|
| 41 |
-
# ==========================================
|
| 42 |
-
# 2. LOAD THE FAST 3B MODEL
|
| 43 |
-
# ==========================================
|
| 44 |
-
print("Loading Unsloth 3B Model...")
|
| 45 |
-
model, tokenizer = FastLanguageModel.from_pretrained(
|
| 46 |
-
model_name = "unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
|
| 47 |
-
max_seq_length = 512,
|
| 48 |
-
dtype = None,
|
| 49 |
-
load_in_4bit = True,
|
| 50 |
-
token = hf_token, # Explicitly passing the token here
|
| 51 |
-
)
|
| 52 |
-
|
| 53 |
-
# Apply LoRA adapters
|
| 54 |
-
model = FastLanguageModel.get_peft_model(
|
| 55 |
-
model,
|
| 56 |
-
r = 16,
|
| 57 |
-
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
|
| 58 |
-
lora_alpha = 16,
|
| 59 |
-
lora_dropout = 0,
|
| 60 |
-
bias = "none",
|
| 61 |
-
use_gradient_checkpointing = "unsloth",
|
| 62 |
-
random_state = 3407,
|
| 63 |
-
)
|
| 64 |
-
|
| 65 |
-
# ==========================================
|
| 66 |
-
# 3. GENERATE ROLLOUTS (BULLETPROOF PARSER)
|
| 67 |
-
# ==========================================
|
| 68 |
-
print("Initializing OpenEnv...")
|
| 69 |
-
env = EnterpriseEmailEnv()
|
| 70 |
-
|
| 71 |
-
def format_observation_to_prompt(obs):
|
| 72 |
-
current_email = obs.get('current_email', {})
|
| 73 |
-
email_id = current_email.get('email_id', 'UNKNOWN_ID')
|
| 74 |
-
sender = current_email.get('sender', 'Unknown')
|
| 75 |
-
subject = current_email.get('subject', 'No Subject')
|
| 76 |
-
body = current_email.get('body', '')[:1000]
|
| 77 |
-
|
| 78 |
-
prompt = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|>
|
| 79 |
-
You are a Corporate Email Triage Agent. You must respond ONLY with a JSON object.
|
| 80 |
-
### AVAILABLE TOOLS:
|
| 81 |
-
1. {{"tool": "route_to_human", "arguments": {{"email_id": "{email_id}", "department": "IT/Support/HR"}}}}
|
| 82 |
-
*RULE: Use this for VIPs, Angry customers, Server Outages, or Sensitive HR issues.*
|
| 83 |
-
2. {{"tool": "auto_reply", "arguments": {{"email_id": "{email_id}", "message": "REPLY_TEXT"}}}}
|
| 84 |
-
*RULE: Use this for routine tasks like password resets or feature requests.*
|
| 85 |
-
3. {{"tool": "ask_for_clarification", "arguments": {{"email_id": "{email_id}"}}}}
|
| 86 |
-
*RULE: Use this only if the email is confusing or missing data.*
|
| 87 |
-
Always use the correct Email ID: {email_id}<|eot_id|><|start_header_id|>user<|end_header_id|>
|
| 88 |
-
Email ID: {email_id}
|
| 89 |
-
From: {sender}
|
| 90 |
-
Subject: {subject}
|
| 91 |
-
Body: {body}<|eot_id|><|start_header_id|>assistant<|end_header_id|>"""
|
| 92 |
-
return prompt
|
| 93 |
-
|
| 94 |
-
def collect_rollouts(env, model, tokenizer, num_episodes=30):
|
| 95 |
-
rollout_data = []
|
| 96 |
-
FastLanguageModel.for_inference(model)
|
| 97 |
-
|
| 98 |
-
for episode in tqdm(range(num_episodes), desc="Collecting rollouts"):
|
| 99 |
-
obs = env.reset()
|
| 100 |
-
episode_data = []
|
| 101 |
-
|
| 102 |
-
while not env.current_step >= env.max_steps and len(env.processed_emails) < len(env.emails):
|
| 103 |
-
prompt = format_observation_to_prompt(obs)
|
| 104 |
-
inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=2048).to(model.device)
|
| 105 |
-
|
| 106 |
-
with torch.no_grad():
|
| 107 |
-
outputs = model.generate(
|
| 108 |
-
**inputs,
|
| 109 |
-
max_new_tokens=300,
|
| 110 |
-
temperature=0.1,
|
| 111 |
-
do_sample=True,
|
| 112 |
-
pad_token_id=tokenizer.eos_token_id,
|
| 113 |
-
)
|
| 114 |
-
|
| 115 |
-
generated_text = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
|
| 116 |
-
|
| 117 |
-
start_idx = generated_text.find('{')
|
| 118 |
-
end_idx = generated_text.rfind('}') + 1
|
| 119 |
-
|
| 120 |
-
if start_idx != -1 and end_idx != 0:
|
| 121 |
-
clean_json = generated_text[start_idx:end_idx]
|
| 122 |
-
try:
|
| 123 |
-
action = json.loads(clean_json)
|
| 124 |
-
if "tool" not in action or "arguments" not in action:
|
| 125 |
-
action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
|
| 126 |
-
except:
|
| 127 |
-
action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
|
| 128 |
-
else:
|
| 129 |
-
action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
|
| 130 |
-
|
| 131 |
-
next_obs, reward, done, info = env.step(action)
|
| 132 |
-
|
| 133 |
-
episode_data.append({
|
| 134 |
-
"prompt": prompt,
|
| 135 |
-
"action": action,
|
| 136 |
-
"reward": reward,
|
| 137 |
-
"observation": next_obs
|
| 138 |
-
})
|
| 139 |
-
|
| 140 |
-
print(f"Step {env.current_step}: {action['tool']} -> Reward: {reward:+.2f} | Total: {env.total_reward:+.2f}")
|
| 141 |
-
obs = next_obs
|
| 142 |
-
if done: break
|
| 143 |
-
|
| 144 |
-
rollout_data.extend(episode_data)
|
| 145 |
-
|
| 146 |
-
FastLanguageModel.for_training(model)
|
| 147 |
-
return rollout_data
|
| 148 |
-
|
| 149 |
-
print("Collecting rollouts (playing the game)...")
|
| 150 |
-
rollout_data = collect_rollouts(env, model, tokenizer, num_episodes=30)
|
| 151 |
-
print(f"Collected {len(rollout_data)} transitions")
|
| 152 |
-
|
| 153 |
-
# ==========================================
|
| 154 |
-
# 4. PREPARE BEHAVIORAL CLONING DATASET
|
| 155 |
-
# ==========================================
|
| 156 |
-
print("Filtering for successful behaviors...")
|
| 157 |
-
winning_rollouts = [item for item in rollout_data if item["reward"] > 0]
|
| 158 |
-
print(f"Found {len(winning_rollouts)} highly successful actions to train on!")
|
| 159 |
-
|
| 160 |
-
sft_dataset_dict = {
|
| 161 |
-
"text": [item["prompt"] + json.dumps(item["action"]) + tokenizer.eos_token for item in winning_rollouts]
|
| 162 |
-
}
|
| 163 |
-
sft_dataset = Dataset.from_dict(sft_dataset_dict)
|
| 164 |
-
|
| 165 |
-
# ==========================================
|
| 166 |
-
# 5. INITIALIZE SFT TRAINER
|
| 167 |
-
# ==========================================
|
| 168 |
-
print("Configuring SFT Trainer...")
|
| 169 |
-
sft_config = TrainingArguments(
|
| 170 |
-
output_dir="email-triage-lora-final",
|
| 171 |
-
per_device_train_batch_size=4,
|
| 172 |
-
gradient_accumulation_steps=4,
|
| 173 |
-
learning_rate=2e-4,
|
| 174 |
-
logging_steps=1,
|
| 175 |
-
max_steps=60,
|
| 176 |
-
optim="adamw_8bit",
|
| 177 |
-
fp16=not torch.cuda.is_bf16_supported(),
|
| 178 |
-
bf16=torch.cuda.is_bf16_supported(),
|
| 179 |
-
)
|
| 180 |
-
|
| 181 |
-
sft_trainer = SFTTrainer(
|
| 182 |
-
model=model,
|
| 183 |
-
tokenizer=tokenizer,
|
| 184 |
-
train_dataset=sft_dataset,
|
| 185 |
-
dataset_text_field="text",
|
| 186 |
-
max_seq_length=512,
|
| 187 |
-
args=sft_config,
|
| 188 |
-
)
|
| 189 |
-
|
| 190 |
-
# ==========================================
|
| 191 |
-
# 6. EXECUTE TRAINING & SAVE DELIVERABLES
|
| 192 |
-
# ==========================================
|
| 193 |
-
print("Starting Behavioral Cloning Training...")
|
| 194 |
-
sft_trainer.train()
|
| 195 |
-
|
| 196 |
-
print("Training completed! Saving LoRA adapters...")
|
| 197 |
-
model.save_pretrained("email-triage-lora-final")
|
| 198 |
-
tokenizer.save_pretrained("email-triage-lora-final")
|
| 199 |
-
print("SUCCESS: Model weights saved locally!")
|
| 200 |
-
|
| 201 |
-
# ==========================================
|
| 202 |
-
# 7. GENERATE HACKATHON EVIDENCE (PLOTS)
|
| 203 |
-
# ==========================================
|
| 204 |
-
print("Generating validation plots...")
|
| 205 |
-
|
| 206 |
-
# Plot 1: Training Loss
|
| 207 |
-
try:
|
| 208 |
-
history = sft_trainer.state.log_history
|
| 209 |
-
steps = [x['step'] for x in history if 'loss' in x]
|
| 210 |
-
loss = [x['loss'] for x in history if 'loss' in x]
|
| 211 |
-
|
| 212 |
-
if loss:
|
| 213 |
-
plt.figure(figsize=(10, 5))
|
| 214 |
-
plt.plot(steps, loss, color='#1f77b4', linewidth=2, label='Training Loss')
|
| 215 |
-
plt.title('Agent Training Loss: Enterprise Email Triage')
|
| 216 |
-
plt.xlabel('Step')
|
| 217 |
-
plt.ylabel('Loss')
|
| 218 |
-
plt.grid(True, linestyle='--', alpha=0.6)
|
| 219 |
-
plt.legend()
|
| 220 |
-
plt.savefig('training_loss.png')
|
| 221 |
-
print("✅ Saved training_loss.png")
|
| 222 |
-
except Exception as e:
|
| 223 |
-
print(f"⚠️ Could not generate loss curve: {e}")
|
| 224 |
-
|
| 225 |
-
# Plot 2: Reward Distribution
|
| 226 |
-
try:
|
| 227 |
-
rewards = [item['reward'] for item in rollout_data]
|
| 228 |
-
if rewards:
|
| 229 |
-
plt.figure(figsize=(10, 5))
|
| 230 |
-
plt.hist(rewards, bins=10, color='green', alpha=0.7, edgecolor='black')
|
| 231 |
-
plt.title('Expert Rollout Reward Distribution')
|
| 232 |
-
plt.xlabel('Reward Value')
|
| 233 |
-
plt.ylabel('Frequency')
|
| 234 |
-
plt.axvline(0, color='red', linestyle='--')
|
| 235 |
-
plt.savefig('reward_curve.png')
|
| 236 |
-
print("✅ Saved reward_curve.png")
|
| 237 |
-
except Exception as e:
|
| 238 |
-
print(f"⚠️ Could not generate reward curve: {e}")
|
| 239 |
-
|
| 240 |
-
print("🎉 ALL TASKS COMPLETE. Script is ready for judging!")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|