TRL: inject text/JSON tool calls + forced ls into tool loop

kaggle/train_grpo.py

@@ -282,6 +282,90 @@ def _parse_tool_calls(text: str) -> list[dict[str, Any]]:
     return calls
 
 
+def _parsed_to_openai_tool_calls(parsed: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    out = []
+    for i, p in enumerate(parsed):
+        args = p.get("arguments") if isinstance(p.get("arguments"), dict) else {}
+        out.append(
+            {
+                "id": f"call_text_{i}",
+                "type": "function",
+                "function": {
+                    "name": str(p["name"]),
+                    "arguments": args,
+                },
+            }
+        )
+    return out
+
+
+def _inject_text_tool_calls_into_completions(
+    completions: list[Any],
+    *,
+    force_ls_if_empty: bool = True,
+) -> tuple[list[Any], int]:
+    """TRL only runs tools from native ``tool_calls``. Qwen often emits markdown JSON — promote it."""
+    injected = 0
+    fixed: list[Any] = []
+    for comp in completions:
+        # conversational: [[{role, content, tool_calls?}]]
+        if (
+            isinstance(comp, list)
+            and comp
+            and isinstance(comp[0], dict)
+            and comp[0].get("role") == "assistant"
+        ):
+            msg = dict(comp[0])
+            existing = msg.get("tool_calls") or []
+            if existing:
+                fixed.append([msg] + list(comp[1:]))
+                continue
+            content = str(msg.get("content") or "")
+            parsed = _parse_tool_calls(content)
+            if not parsed and force_ls_if_empty:
+                # Last resort so the GRPO group is not all "DONE"/no-tools.
+                parsed = [{"name": "ls", "arguments": {"path": "."}}]
+                msg["content"] = (content + "\n").strip() + "\n(forced tool: ls path='.')"
+            if parsed:
+                msg["tool_calls"] = _parsed_to_openai_tool_calls(parsed)
+                injected += 1
+                _vlog(
+                    "train.inject_text_tool_calls",
+                    n=len(parsed),
+                    names=[p["name"] for p in parsed],
+                    forced=not bool(_parse_tool_calls(content)),
+                )
+            fixed.append([msg] + list(comp[1:]))
+        else:
+            fixed.append(comp)
+    return fixed, injected
+
+
+def make_tool_repair_grpo_trainer(base_cls: type):
+    """Subclass GRPOTrainer to execute markdown/JSON tool blobs + forced ls."""
+
+    class ToolRepairGRPOTrainer(base_cls):  # type: ignore[misc,valid-type]
+        def _tool_call_loop(self, prompts, prompt_ids, completion_ids, completions, logprobs, images, multimodal_fields):
+            force = os.getenv("DARL_FORCE_FIRST_TOOL", "1").strip() not in ("0", "false", "no")
+            completions, n_inj = _inject_text_tool_calls_into_completions(
+                completions, force_ls_if_empty=force
+            )
+            if n_inj:
+                _vlog("train.tool_repair.injected", n=n_inj, force=force)
+            return super()._tool_call_loop(
+                prompts,
+                prompt_ids,
+                completion_ids,
+                completions,
+                logprobs,
+                images,
+                multimodal_fields,
+            )
+
+    ToolRepairGRPOTrainer.__name__ = "ToolRepairGRPOTrainer"
+    return ToolRepairGRPOTrainer
+
+
 def run_heldout_agent_eval(
     model,
     tokenizer,
@@ -1385,14 +1469,16 @@ def main(cfg: Cfg | None = None) -> None:
         max_eval_turns=cfg.max_eval_turns,
     )
 
+    TrainerCls = make_tool_repair_grpo_trainer(GRPOTrainer)
     _vlog(
         "main.trainer_init",
         environment="DeepAgentSWEEnv",
+        trainer=TrainerCls.__name__,
         max_steps=cfg.max_steps,
         num_generations=cfg.num_generations,
         max_tool_calling_iterations=cfg.max_tool_calling_iterations,
     )
-    trainer = GRPOTrainer(
+    trainer = TrainerCls(
         model=model,
         reward_funcs=[log_completions_reward],  # logs only; env get_reward still scores SWE
         args=args,