Proteinrequired commited on
Commit
74a5ecc
·
verified ·
1 Parent(s): 3907ccb

Delete train.py

Browse files
Files changed (1) hide show
  1. train.py +0 -240
train.py DELETED
@@ -1,240 +0,0 @@
1
- # ==========================================
2
- # 0. SIMPLE AUTHENTICATION (FOR GATED MODEL)
3
- # ==========================================
4
- import os
5
- import sys
6
- from huggingface_hub import login
7
-
8
- # Grab the secret you set in the HF Space UI
9
- hf_token = os.environ.get("HF_TOKEN")
10
-
11
- if not hf_token:
12
- print("❌ ERROR: HF_TOKEN not found in Secrets. Llama-3.2 download will fail.")
13
- sys.exit(1)
14
-
15
- # Silently log in so Unsloth can download the gated weights
16
- login(token=hf_token.strip())
17
- print("✅ Authenticated. Ready to load Llama-3.2.")
18
-
19
- # ==========================================
20
- # 1. THE STABLE ENVIRONMENT INSTALL
21
- # ==========================================
22
- # (Commented out for local runs, judges will run these if necessary)
23
- import os
24
- os.system('pip install "torch>=2.4.0,<2.11.0" torchvision torchaudio')
25
- os.system('pip install unsloth unsloth_zoo')
26
- os.system('pip install "trl==0.22.0" peft accelerate bitsandbytes openenv-core datasets matplotlib pandas')
27
-
28
- import re
29
- import json
30
- import torch
31
- import matplotlib.pyplot as plt
32
- from tqdm import tqdm
33
- from datasets import Dataset
34
- from unsloth import FastLanguageModel
35
- from trl import SFTTrainer
36
- from transformers import TrainingArguments
37
-
38
- # Import the environment we just ensured exists
39
- from env import EnterpriseEmailEnv
40
-
41
- # ==========================================
42
- # 2. LOAD THE FAST 3B MODEL
43
- # ==========================================
44
- print("Loading Unsloth 3B Model...")
45
- model, tokenizer = FastLanguageModel.from_pretrained(
46
- model_name = "unsloth/Llama-3.2-3B-Instruct-bnb-4bit",
47
- max_seq_length = 512,
48
- dtype = None,
49
- load_in_4bit = True,
50
- token = hf_token, # Explicitly passing the token here
51
- )
52
-
53
- # Apply LoRA adapters
54
- model = FastLanguageModel.get_peft_model(
55
- model,
56
- r = 16,
57
- target_modules = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
58
- lora_alpha = 16,
59
- lora_dropout = 0,
60
- bias = "none",
61
- use_gradient_checkpointing = "unsloth",
62
- random_state = 3407,
63
- )
64
-
65
- # ==========================================
66
- # 3. GENERATE ROLLOUTS (BULLETPROOF PARSER)
67
- # ==========================================
68
- print("Initializing OpenEnv...")
69
- env = EnterpriseEmailEnv()
70
-
71
- def format_observation_to_prompt(obs):
72
- current_email = obs.get('current_email', {})
73
- email_id = current_email.get('email_id', 'UNKNOWN_ID')
74
- sender = current_email.get('sender', 'Unknown')
75
- subject = current_email.get('subject', 'No Subject')
76
- body = current_email.get('body', '')[:1000]
77
-
78
- prompt = f"""<|begin_of_text|><|start_header_id|>system<|end_header_id|>
79
- You are a Corporate Email Triage Agent. You must respond ONLY with a JSON object.
80
- ### AVAILABLE TOOLS:
81
- 1. {{"tool": "route_to_human", "arguments": {{"email_id": "{email_id}", "department": "IT/Support/HR"}}}}
82
- *RULE: Use this for VIPs, Angry customers, Server Outages, or Sensitive HR issues.*
83
- 2. {{"tool": "auto_reply", "arguments": {{"email_id": "{email_id}", "message": "REPLY_TEXT"}}}}
84
- *RULE: Use this for routine tasks like password resets or feature requests.*
85
- 3. {{"tool": "ask_for_clarification", "arguments": {{"email_id": "{email_id}"}}}}
86
- *RULE: Use this only if the email is confusing or missing data.*
87
- Always use the correct Email ID: {email_id}<|eot_id|><|start_header_id|>user<|end_header_id|>
88
- Email ID: {email_id}
89
- From: {sender}
90
- Subject: {subject}
91
- Body: {body}<|eot_id|><|start_header_id|>assistant<|end_header_id|>"""
92
- return prompt
93
-
94
- def collect_rollouts(env, model, tokenizer, num_episodes=30):
95
- rollout_data = []
96
- FastLanguageModel.for_inference(model)
97
-
98
- for episode in tqdm(range(num_episodes), desc="Collecting rollouts"):
99
- obs = env.reset()
100
- episode_data = []
101
-
102
- while not env.current_step >= env.max_steps and len(env.processed_emails) < len(env.emails):
103
- prompt = format_observation_to_prompt(obs)
104
- inputs = tokenizer(prompt, return_tensors="pt", padding=True, truncation=True, max_length=2048).to(model.device)
105
-
106
- with torch.no_grad():
107
- outputs = model.generate(
108
- **inputs,
109
- max_new_tokens=300,
110
- temperature=0.1,
111
- do_sample=True,
112
- pad_token_id=tokenizer.eos_token_id,
113
- )
114
-
115
- generated_text = tokenizer.decode(outputs[0][inputs.input_ids.shape[-1]:], skip_special_tokens=True)
116
-
117
- start_idx = generated_text.find('{')
118
- end_idx = generated_text.rfind('}') + 1
119
-
120
- if start_idx != -1 and end_idx != 0:
121
- clean_json = generated_text[start_idx:end_idx]
122
- try:
123
- action = json.loads(clean_json)
124
- if "tool" not in action or "arguments" not in action:
125
- action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
126
- except:
127
- action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
128
- else:
129
- action = {"tool": "ask_for_clarification", "arguments": {"email_id": obs['current_email']['email_id']}}
130
-
131
- next_obs, reward, done, info = env.step(action)
132
-
133
- episode_data.append({
134
- "prompt": prompt,
135
- "action": action,
136
- "reward": reward,
137
- "observation": next_obs
138
- })
139
-
140
- print(f"Step {env.current_step}: {action['tool']} -> Reward: {reward:+.2f} | Total: {env.total_reward:+.2f}")
141
- obs = next_obs
142
- if done: break
143
-
144
- rollout_data.extend(episode_data)
145
-
146
- FastLanguageModel.for_training(model)
147
- return rollout_data
148
-
149
- print("Collecting rollouts (playing the game)...")
150
- rollout_data = collect_rollouts(env, model, tokenizer, num_episodes=30)
151
- print(f"Collected {len(rollout_data)} transitions")
152
-
153
- # ==========================================
154
- # 4. PREPARE BEHAVIORAL CLONING DATASET
155
- # ==========================================
156
- print("Filtering for successful behaviors...")
157
- winning_rollouts = [item for item in rollout_data if item["reward"] > 0]
158
- print(f"Found {len(winning_rollouts)} highly successful actions to train on!")
159
-
160
- sft_dataset_dict = {
161
- "text": [item["prompt"] + json.dumps(item["action"]) + tokenizer.eos_token for item in winning_rollouts]
162
- }
163
- sft_dataset = Dataset.from_dict(sft_dataset_dict)
164
-
165
- # ==========================================
166
- # 5. INITIALIZE SFT TRAINER
167
- # ==========================================
168
- print("Configuring SFT Trainer...")
169
- sft_config = TrainingArguments(
170
- output_dir="email-triage-lora-final",
171
- per_device_train_batch_size=4,
172
- gradient_accumulation_steps=4,
173
- learning_rate=2e-4,
174
- logging_steps=1,
175
- max_steps=60,
176
- optim="adamw_8bit",
177
- fp16=not torch.cuda.is_bf16_supported(),
178
- bf16=torch.cuda.is_bf16_supported(),
179
- )
180
-
181
- sft_trainer = SFTTrainer(
182
- model=model,
183
- tokenizer=tokenizer,
184
- train_dataset=sft_dataset,
185
- dataset_text_field="text",
186
- max_seq_length=512,
187
- args=sft_config,
188
- )
189
-
190
- # ==========================================
191
- # 6. EXECUTE TRAINING & SAVE DELIVERABLES
192
- # ==========================================
193
- print("Starting Behavioral Cloning Training...")
194
- sft_trainer.train()
195
-
196
- print("Training completed! Saving LoRA adapters...")
197
- model.save_pretrained("email-triage-lora-final")
198
- tokenizer.save_pretrained("email-triage-lora-final")
199
- print("SUCCESS: Model weights saved locally!")
200
-
201
- # ==========================================
202
- # 7. GENERATE HACKATHON EVIDENCE (PLOTS)
203
- # ==========================================
204
- print("Generating validation plots...")
205
-
206
- # Plot 1: Training Loss
207
- try:
208
- history = sft_trainer.state.log_history
209
- steps = [x['step'] for x in history if 'loss' in x]
210
- loss = [x['loss'] for x in history if 'loss' in x]
211
-
212
- if loss:
213
- plt.figure(figsize=(10, 5))
214
- plt.plot(steps, loss, color='#1f77b4', linewidth=2, label='Training Loss')
215
- plt.title('Agent Training Loss: Enterprise Email Triage')
216
- plt.xlabel('Step')
217
- plt.ylabel('Loss')
218
- plt.grid(True, linestyle='--', alpha=0.6)
219
- plt.legend()
220
- plt.savefig('training_loss.png')
221
- print("✅ Saved training_loss.png")
222
- except Exception as e:
223
- print(f"⚠️ Could not generate loss curve: {e}")
224
-
225
- # Plot 2: Reward Distribution
226
- try:
227
- rewards = [item['reward'] for item in rollout_data]
228
- if rewards:
229
- plt.figure(figsize=(10, 5))
230
- plt.hist(rewards, bins=10, color='green', alpha=0.7, edgecolor='black')
231
- plt.title('Expert Rollout Reward Distribution')
232
- plt.xlabel('Reward Value')
233
- plt.ylabel('Frequency')
234
- plt.axvline(0, color='red', linestyle='--')
235
- plt.savefig('reward_curve.png')
236
- print("✅ Saved reward_curve.png")
237
- except Exception as e:
238
- print(f"⚠️ Could not generate reward curve: {e}")
239
-
240
- print("🎉 ALL TASKS COMPLETE. Script is ready for judging!")