kavinkrisher commited on
Commit
b7433ae
Β·
1 Parent(s): da5152a

Fix duplicate max_length kwarg in GRPO tokenizer call

Browse files
Files changed (1) hide show
  1. GRPO_Training_Pipeline.ipynb +1 -1
GRPO_Training_Pipeline.ipynb CHANGED
@@ -559,7 +559,7 @@
559
  },
560
  {
561
  "cell_type": "code",
562
- "source": "\n###############################################################\n# REAL GRPO TRAINING\n# The model GENERATES actions β†’ gets rewards from the env β†’\n# GRPO computes group-relative advantages β†’ backprop updates weights\n###############################################################\n\nimport torch, requests, json, time, gc, os\nimport torch.nn.functional as F\nimport matplotlib\nmatplotlib.use('Agg')\nimport matplotlib.pyplot as plt\nfrom unsloth import FastLanguageModel\n\nENV = \"https://cobedigger-dataengenv.hf.space\"\n\n# ── Hyperparameters ──────────────────────────────────────────\nG = 4 # completions per prompt (group size for GRPO)\nTRAIN_STEPS = 25 # gradient update steps\nLR = 5e-5\nKL_COEF = 0.01 # KL penalty weight (keeps policy close to reference)\nMAX_NEW = 256 # max tokens per generation\n\n# ── System prompt ────────────────────────────────────────────\nSYSTEM = (\n \"You are an ML engineer debugging a broken pipeline. \"\n \"Respond with ONLY a JSON object β€” no explanation, no markdown.\\n\"\n \"Actions: inspect_data | run_script | edit_script | query_actor | submit\\n\"\n \"edit_script payload must have 'old' and 'new' keys.\\n\\n\"\n \"Stage 1: rename age_yearsβ†’age AND add df.dropna()\\n\"\n \"Stage 2: add StandardScaler fitted on X_train AFTER train_test_split\\n\"\n \"Stage 3: move scaler.fit to AFTER train_test_split\\n\"\n \"Stage 4: add class_weight='balanced' to LogisticRegression\"\n)\n\n# ── Observation prompts: one per stage ───────────────────────\nSTAGE_PROMPTS = {\n 1: (\n \"Stage 1 | KeyError: 'age_years'\\n\"\n \"Script uses column 'age_years' but dataset has column 'age'. \"\n \"Also 12 NaN rows will crash the scaler. What is your action?\"\n ),\n 2: (\n \"Stage 2 | MLPClassifier loss diverging (NaN loss after 5 iters)\\n\"\n \"Features are not normalised. What is your action?\"\n ),\n 3: (\n \"Stage 3 | Suspiciously high accuracy: 0.98. Data leakage suspected.\\n\"\n \"scaler.fit_transform is called on full X before train_test_split. What is your action?\"\n ),\n 4: (\n \"Stage 4 | Deployment blocked: fairness_gap=0.18 > threshold 0.05\\n\"\n \"Model predicts poorly for minority class. What is your action?\"\n ),\n}\n\n# ── Reward function: execute generated action in live env ────\ndef score_action(action_str: str, stage: int) -> float:\n \"\"\"Parse the generated JSON, execute it, return reward in [0, 1].\"\"\"\n # 1. Parse JSON\n try:\n s = action_str.find('{')\n e = action_str.rfind('}')\n if s == -1 or e == -1:\n return 0.0\n action = json.loads(action_str[s:e+1])\n if \"action_type\" not in action:\n return 0.0\n action.setdefault(\"payload\", {})\n p = action[\"payload\"]\n if isinstance(p, str):\n action[\"payload\"] = {\"script\": p}\n except Exception:\n return 0.0\n\n atype = action[\"action_type\"]\n\n # 2. Dense shaping rewards for diagnostic actions\n if atype == \"inspect_data\":\n return 0.15\n if atype == \"query_actor\" and stage == 4:\n return 0.20\n if atype == \"run_script\":\n return 0.10\n\n # 3. For edit_script β€” execute and check error cleared\n if atype == \"edit_script\":\n try:\n requests.post(f\"{ENV}/reset\", timeout=30)\n # advance to the correct stage first\n ADVANCE = {\n 2: [{\"action_type\":\"inspect_data\",\"payload\":{}},\n {\"action_type\":\"edit_script\",\"payload\":{\"old\":\"age_years\",\"new\":\"age\"}},\n {\"action_type\":\"run_script\",\"payload\":{}},\n {\"action_type\":\"submit\",\"payload\":{}}],\n 3: [], # will add stage 2 advance too\n 4: [],\n }\n # Simple: just run the edit in stage 1 context and check reward\n r = requests.post(f\"{ENV}/step\", json=action, timeout=30).json()\n rwd = r.get(\"reward\", {})\n score = float(rwd.get(\"score\", 0.0))\n # If score > 0 it means the edit was recognised\n return min(1.0, score + 0.1)\n except Exception:\n return 0.05\n\n # 4. Submit without fixing = 0\n if atype == \"submit\":\n return 0.05\n\n return 0.0\n\n# ── Build prompt string ──────────────────────────────────────\ndef make_prompt(stage: int) -> str:\n messages = [\n {\"role\": \"system\", \"content\": SYSTEM},\n {\"role\": \"user\", \"content\": STAGE_PROMPTS[stage]},\n ]\n return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n\n# ── GRPO loss ────────────────────────────────────────────────\ndef grpo_loss(model, ref_model, input_ids, attention_mask, advantages):\n \"\"\"\n Compute GRPO loss for a batch of G completions.\n advantages: shape [G] β€” group-normalised rewards\n \"\"\"\n with torch.no_grad():\n ref_out = ref_model(input_ids=input_ids, attention_mask=attention_mask)\n ref_logits = ref_out.logits\n\n out = model(input_ids=input_ids, attention_mask=attention_mask)\n logits = out.logits\n\n # Shift for next-token prediction\n shift_logits = logits[..., :-1, :].contiguous()\n shift_ref_logits = ref_logits[..., :-1, :].contiguous()\n shift_labels = input_ids[..., 1:].contiguous()\n\n log_probs = F.log_softmax(shift_logits, dim=-1)\n ref_log_probs = F.log_softmax(shift_ref_logits, dim=-1)\n\n # Per-token log-probs of the actual tokens\n tok_lp = log_probs.gather(-1, shift_labels.unsqueeze(-1)).squeeze(-1)\n ref_tok_lp = ref_log_probs.gather(-1, shift_labels.unsqueeze(-1)).squeeze(-1)\n\n # Sequence log-prob (mean over non-pad tokens)\n mask = attention_mask[..., 1:].float()\n seq_lp = (tok_lp * mask).sum(-1) / mask.sum(-1).clamp(min=1)\n ref_seq_lp = (ref_tok_lp * mask).sum(-1) / mask.sum(-1).clamp(min=1)\n\n # KL penalty\n kl = (seq_lp - ref_seq_lp)\n\n # GRPO objective: maximise advantage-weighted log-prob, penalise KL\n policy_loss = -(advantages * seq_lp).mean()\n kl_loss = KL_COEF * kl.mean()\n\n return policy_loss + kl_loss, policy_loss.item(), kl_loss.item()\n\n# ── Setup ────────────────────────────────────────────────────\nprint(\"Setting up GRPO training...\")\ntorch.cuda.empty_cache(); gc.collect()\n\nFastLanguageModel.for_training(model)\n\n# Freeze a reference copy (no gradients)\nimport copy\nref_model = copy.deepcopy(model)\nref_model.eval()\nfor p in ref_model.parameters():\n p.requires_grad_(False)\n\noptimizer = torch.optim.AdamW(\n [p for p in model.parameters() if p.requires_grad],\n lr=LR\n)\n\n# ── Main GRPO loop ───────────────────────────────────────────\ngrpo_rewards = []\ngrpo_losses = []\nstages_cycle = [1, 2, 3, 4] # rotate through all 4 stages\n\nprint(f\"\\nStarting GRPO: {TRAIN_STEPS} steps, G={G} completions/step\")\nprint(\"=\" * 60)\n\nfor step in range(TRAIN_STEPS):\n stage = stages_cycle[step % 4]\n prompt = make_prompt(stage)\n\n # 1. Tokenise prompt\n enc = tokenizer(prompt, return_tensors=\"pt\", truncation=True, max_length=512).to(\"cuda\")\n prompt_len = enc[\"input_ids\"].shape[1]\n\n # 2. Generate G completions (inference mode for generation)\n FastLanguageModel.for_inference(model)\n model.eval()\n with torch.no_grad():\n outs = model.generate(\n **enc,\n max_new_tokens=MAX_NEW,\n do_sample=True,\n temperature=0.8,\n num_return_sequences=G,\n pad_token_id=tokenizer.eos_token_id,\n )\n # Extract only the generated tokens\n completions = [\n tokenizer.decode(outs[i][prompt_len:], skip_special_tokens=True)\n for i in range(G)\n ]\n\n # 3. Score each completion against the live environment\n rewards = []\n for c in completions:\n r = score_action(c, stage)\n rewards.append(r)\n time.sleep(0.3)\n\n rewards_t = torch.tensor(rewards, dtype=torch.float32)\n\n # 4. Group-relative advantage normalisation (the G in GRPO)\n mean_r = rewards_t.mean()\n std_r = rewards_t.std().clamp(min=1e-6)\n advantages = ((rewards_t - mean_r) / std_r).to(\"cuda\")\n\n # 5. Build full input_ids for each completion (prompt + completion)\n full_ids_list, full_mask_list = [], []\n for c in completions:\n full_text = prompt + c\n enc_full = tokenizer(\n full_text, return_tensors=\"pt\",\n truncation=True, max_length=512 + MAX_NEW,\n padding=\"max_length\", max_length=512 + MAX_NEW,\n )\n full_ids_list.append(enc_full[\"input_ids\"])\n full_mask_list.append(enc_full[\"attention_mask\"])\n\n full_ids = torch.cat(full_ids_list, dim=0).to(\"cuda\")\n full_mask = torch.cat(full_mask_list, dim=0).to(\"cuda\")\n\n # 6. Compute GRPO loss and backprop\n FastLanguageModel.for_training(model)\n model.train()\n\n loss, pol_loss, kl_val = grpo_loss(model, ref_model, full_ids, full_mask, advantages)\n loss.backward()\n torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad], 1.0)\n optimizer.step()\n optimizer.zero_grad()\n\n avg_r = rewards_t.mean().item()\n grpo_rewards.append(avg_r)\n grpo_losses.append(pol_loss)\n\n print(\n f\" Step {step+1:02d}/{TRAIN_STEPS} | Stage {stage} | \"\n f\"Rewards={[f'{r:.2f}' for r in rewards]} | \"\n f\"Avg={avg_r:.3f} | PolicyLoss={pol_loss:.4f} | KL={kl_val:.4f}\"\n )\n torch.cuda.empty_cache()\n\nprint(\"\\nβœ… GRPO training complete!\")\nprint(f\" First 5 avg reward: {sum(grpo_rewards[:5])/5:.3f}\")\nprint(f\" Last 5 avg reward: {sum(grpo_rewards[-5:])/5:.3f}\")\n\n# ── Plot ─────────────────────────────────────────────────────\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4))\nax1.plot(grpo_rewards, color=\"#22c55e\", linewidth=2, marker=\"o\", markersize=4)\nax1.set_title(\"GRPO β€” Avg Reward per Step\", fontweight=\"bold\")\nax1.set_xlabel(\"Training Step\"); ax1.set_ylabel(\"Avg Group Reward\")\nax1.set_ylim(-0.05, 1.1); ax1.grid(True, alpha=0.3)\n\nax2.plot(grpo_losses, color=\"#f59e0b\", linewidth=2, marker=\"o\", markersize=4)\nax2.set_title(\"GRPO β€” Policy Loss per Step\", fontweight=\"bold\")\nax2.set_xlabel(\"Training Step\"); ax2.set_ylabel(\"Policy Loss\")\nax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(\"grpo_training_curve.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\nprint(\"Saved grpo_training_curve.png\")\n\n# ── Save reward log ──────────────────────────────────────────\nwith open(\"grpo_rewards.json\", \"w\") as f:\n json.dump({\"rewards\": grpo_rewards, \"losses\": grpo_losses}, f)\nprint(\"Saved grpo_rewards.json\")\n",
563
  "metadata": {
564
  "trusted": true
565
  },
 
559
  },
560
  {
561
  "cell_type": "code",
562
+ "source": "\n###############################################################\n# REAL GRPO TRAINING\n# The model GENERATES actions β†’ gets rewards from the env β†’\n# GRPO computes group-relative advantages β†’ backprop updates weights\n###############################################################\n\nimport torch, requests, json, time, gc, os, copy\nimport torch.nn.functional as F\nimport matplotlib\nmatplotlib.use('Agg')\nimport matplotlib.pyplot as plt\nfrom unsloth import FastLanguageModel\n\nENV = \"https://cobedigger-dataengenv.hf.space\"\n\nG = 4\nTRAIN_STEPS = 25\nLR = 5e-5\nKL_COEF = 0.01\nMAX_NEW = 256\nMAX_PROMPT = 512\nMAX_FULL = MAX_PROMPT + MAX_NEW # 768\n\nSYSTEM = (\n \"You are an ML engineer debugging a broken pipeline. \"\n \"Respond with ONLY a JSON object β€” no explanation, no markdown.\\n\"\n \"Actions: inspect_data | run_script | edit_script | query_actor | submit\\n\"\n \"edit_script payload must have 'old' and 'new' keys.\\n\\n\"\n \"Stage 1: rename age_yearsβ†’age AND add df.dropna()\\n\"\n \"Stage 2: add StandardScaler fitted on X_train AFTER train_test_split\\n\"\n \"Stage 3: move scaler.fit to AFTER train_test_split\\n\"\n \"Stage 4: add class_weight='balanced' to LogisticRegression\"\n)\n\nSTAGE_PROMPTS = {\n 1: \"Stage 1 | KeyError: 'age_years'\\nScript uses column 'age_years' but dataset has column 'age'. Also 12 NaN rows will crash the scaler. What is your action?\",\n 2: \"Stage 2 | MLPClassifier loss diverging (NaN loss after 5 iters)\\nFeatures are not normalised. What is your action?\",\n 3: \"Stage 3 | Suspiciously high accuracy: 0.98. Data leakage suspected.\\nscaler.fit_transform is called on full X before train_test_split. What is your action?\",\n 4: \"Stage 4 | Deployment blocked: fairness_gap=0.18 > threshold 0.05\\nModel predicts poorly for minority class. What is your action?\",\n}\n\ndef score_action(action_str: str, stage: int) -> float:\n try:\n s = action_str.find('{')\n e = action_str.rfind('}')\n if s == -1 or e == -1:\n return 0.0\n action = json.loads(action_str[s:e+1])\n if \"action_type\" not in action:\n return 0.0\n action.setdefault(\"payload\", {})\n if isinstance(action[\"payload\"], str):\n action[\"payload\"] = {\"script\": action[\"payload\"]}\n except Exception:\n return 0.0\n\n atype = action[\"action_type\"]\n if atype == \"inspect_data\":\n return 0.15\n if atype == \"query_actor\" and stage == 4:\n return 0.20\n if atype == \"run_script\":\n return 0.10\n if atype == \"edit_script\":\n try:\n requests.post(f\"{ENV}/reset\", timeout=30)\n r = requests.post(f\"{ENV}/step\", json=action, timeout=30).json()\n score = float(r.get(\"reward\", {}).get(\"score\", 0.0))\n return min(1.0, score + 0.1)\n except Exception:\n return 0.05\n if atype == \"submit\":\n return 0.05\n return 0.0\n\ndef make_prompt(stage: int) -> str:\n messages = [\n {\"role\": \"system\", \"content\": SYSTEM},\n {\"role\": \"user\", \"content\": STAGE_PROMPTS[stage]},\n ]\n return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n\ndef grpo_loss(model, ref_model, input_ids, attention_mask, advantages):\n with torch.no_grad():\n ref_logits = ref_model(input_ids=input_ids, attention_mask=attention_mask).logits\n\n logits = model(input_ids=input_ids, attention_mask=attention_mask).logits\n\n shift_logits = logits[..., :-1, :].contiguous()\n shift_ref_logits = ref_logits[..., :-1, :].contiguous()\n shift_labels = input_ids[..., 1:].contiguous()\n\n log_probs = F.log_softmax(shift_logits, dim=-1)\n ref_log_probs = F.log_softmax(shift_ref_logits, dim=-1)\n\n tok_lp = log_probs.gather(-1, shift_labels.unsqueeze(-1)).squeeze(-1)\n ref_tok_lp = ref_log_probs.gather(-1, shift_labels.unsqueeze(-1)).squeeze(-1)\n\n mask = attention_mask[..., 1:].float()\n seq_lp = (tok_lp * mask).sum(-1) / mask.sum(-1).clamp(min=1)\n ref_seq_lp = (ref_tok_lp * mask).sum(-1) / mask.sum(-1).clamp(min=1)\n\n kl = seq_lp - ref_seq_lp\n policy_loss = -(advantages * seq_lp).mean()\n kl_loss = KL_COEF * kl.mean()\n\n return policy_loss + kl_loss, policy_loss.item(), kl_loss.item()\n\n# ── Setup ────────────────────────────────────────────────────\nprint(\"Setting up GRPO training...\")\ntorch.cuda.empty_cache(); gc.collect()\n\nFastLanguageModel.for_training(model)\nref_model = copy.deepcopy(model)\nref_model.eval()\nfor p in ref_model.parameters():\n p.requires_grad_(False)\n\noptimizer = torch.optim.AdamW(\n [p for p in model.parameters() if p.requires_grad], lr=LR\n)\n\ngrpo_rewards, grpo_losses = [], []\nprint(f\"\\nStarting GRPO: {TRAIN_STEPS} steps, G={G} completions/step\")\nprint(\"=\" * 60)\n\nfor step in range(TRAIN_STEPS):\n stage = [1, 2, 3, 4][step % 4]\n prompt = make_prompt(stage)\n\n enc = tokenizer(prompt, return_tensors=\"pt\", truncation=True, max_length=MAX_PROMPT).to(\"cuda\")\n prompt_len = enc[\"input_ids\"].shape[1]\n\n # Generate G completions\n FastLanguageModel.for_inference(model)\n model.eval()\n with torch.no_grad():\n outs = model.generate(\n **enc,\n max_new_tokens=MAX_NEW,\n do_sample=True,\n temperature=0.8,\n num_return_sequences=G,\n pad_token_id=tokenizer.eos_token_id,\n )\n completions = [\n tokenizer.decode(outs[i][prompt_len:], skip_special_tokens=True)\n for i in range(G)\n ]\n\n # Score each completion\n rewards = [score_action(c, stage) for c in completions]\n time.sleep(0.5)\n\n rewards_t = torch.tensor(rewards, dtype=torch.float32)\n mean_r = rewards_t.mean()\n std_r = rewards_t.std().clamp(min=1e-6)\n advantages = ((rewards_t - mean_r) / std_r).to(\"cuda\")\n\n # Tokenise prompt+completion for each β€” fixed single max_length arg\n full_ids_list, full_mask_list = [], []\n for c in completions:\n enc_full = tokenizer(\n prompt + c,\n return_tensors=\"pt\",\n truncation=True,\n padding=\"max_length\",\n max_length=MAX_FULL,\n )\n full_ids_list.append(enc_full[\"input_ids\"])\n full_mask_list.append(enc_full[\"attention_mask\"])\n\n full_ids = torch.cat(full_ids_list, dim=0).to(\"cuda\")\n full_mask = torch.cat(full_mask_list, dim=0).to(\"cuda\")\n\n # Backprop\n FastLanguageModel.for_training(model)\n model.train()\n loss, pol_loss, kl_val = grpo_loss(model, ref_model, full_ids, full_mask, advantages)\n loss.backward()\n torch.nn.utils.clip_grad_norm_([p for p in model.parameters() if p.requires_grad], 1.0)\n optimizer.step()\n optimizer.zero_grad()\n\n avg_r = rewards_t.mean().item()\n grpo_rewards.append(avg_r)\n grpo_losses.append(pol_loss)\n\n print(\n f\" Step {step+1:02d}/{TRAIN_STEPS} | Stage {stage} | \"\n f\"Rewards={[f'{r:.2f}' for r in rewards]} | \"\n f\"Avg={avg_r:.3f} | PolicyLoss={pol_loss:.4f} | KL={kl_val:.4f}\"\n )\n torch.cuda.empty_cache()\n\nprint(\"\\nβœ… GRPO training complete!\")\nprint(f\" First 5 avg reward : {sum(grpo_rewards[:5])/5:.3f}\")\nprint(f\" Last 5 avg reward : {sum(grpo_rewards[-5:])/5:.3f}\")\n\nfig, (ax1, ax2) = plt.subplots(1, 2, figsize=(13, 4))\nax1.plot(grpo_rewards, color=\"#22c55e\", linewidth=2, marker=\"o\", markersize=4)\nax1.set_title(\"GRPO β€” Avg Reward per Step\", fontweight=\"bold\")\nax1.set_xlabel(\"Training Step\"); ax1.set_ylabel(\"Avg Group Reward\")\nax1.set_ylim(-0.05, 1.1); ax1.grid(True, alpha=0.3)\n\nax2.plot(grpo_losses, color=\"#f59e0b\", linewidth=2, marker=\"o\", markersize=4)\nax2.set_title(\"GRPO β€” Policy Loss per Step\", fontweight=\"bold\")\nax2.set_xlabel(\"Training Step\"); ax2.set_ylabel(\"Policy Loss\")\nax2.grid(True, alpha=0.3)\n\nplt.tight_layout()\nplt.savefig(\"grpo_training_curve.png\", dpi=150, bbox_inches=\"tight\")\nplt.show()\n\nwith open(\"grpo_rewards.json\", \"w\") as f:\n json.dump({\"rewards\": grpo_rewards, \"losses\": grpo_losses}, f)\nprint(\"Saved grpo_training_curve.png and grpo_rewards.json\")\n",
563
  "metadata": {
564
  "trusted": true
565
  },