Force first tool + tool_choice required nudge

kaggle/train_grpo.py

@@ -383,8 +383,33 @@ def run_heldout_agent_eval(
                 final_text = completion
                 messages.append({"role": "assistant", "content": completion})
                 if not calls:
-                    _vlog("heldout_eval.stop_no_tools", task_id=task_id, turn=turn)
-                    break
+                    # Force-first-tool: if the model refuses tools on early turns, inject ls(".")
+                    # so the episode can continue and shaped rewards can differentiate.
+                    force = os.getenv("DARL_FORCE_FIRST_TOOL", "1").strip() not in (
+                        "0",
+                        "false",
+                        "no",
+                    )
+                    if force and turn < 2:
+                        _vlog(
+                            "heldout_eval.force_first_tool",
+                            task_id=task_id,
+                            turn=turn,
+                            tool="ls",
+                        )
+                        calls = [{"name": "ls", "arguments": {"path": "."}}]
+                        messages.append(
+                            {
+                                "role": "assistant",
+                                "content": (
+                                    "(system) No tool call detected — forcing ls(path='.') "
+                                    "so you can continue with real paths."
+                                ),
+                            }
+                        )
+                    else:
+                        _vlog("heldout_eval.stop_no_tools", task_id=task_id, turn=turn)
+                        break
                 for call in calls:
                     _vlog(
                         "heldout_eval.tool_dispatch",
@@ -1314,6 +1339,12 @@ def main(cfg: Cfg | None = None) -> None:
         step=0,
     )
 
+    # Nudge Qwen chat template toward required tool calls when supported.
+    chat_kwargs: dict[str, Any] = {"tool_choice": "required"}
+    gen_kwargs: dict[str, Any] = {
+        "temperature": float(os.getenv("DARL_GEN_TEMPERATURE", "0.8")),
+        "top_p": 0.95,
+    }
     args = GRPOConfig(
         output_dir=output_dir,
         run_name="kaggle-grpo-swe",
@@ -1336,6 +1367,13 @@ def main(cfg: Cfg | None = None) -> None:
         hub_model_id=cfg.hub_model_id,
         hub_token=os.environ["HF_TOKEN"],
         push_to_hub=False,
+        chat_template_kwargs=chat_kwargs,
+        generation_kwargs=gen_kwargs,
+    )
+    _vlog(
+        "main.grpo_config_tool_nudge",
+        chat_template_kwargs=chat_kwargs,
+        generation_kwargs=gen_kwargs,
     )
 
     cb = make_eval_before_save_callback(