Force Qwen3 chat template to disable thinking mode via tokenizer patch

Diagnosed the 100% clipped_ratio and entropy=0.07 collapse: Qwen3's chat
template was auto-inserting <think>...</think> prefixes, and the 0.6B
model was burning its entire 512-token completion budget on reasoning
text without ever emitting a tool call. The GRPOConfig flag
chat_template_kwargs={'enable_thinking': False} was silently dropped by
the installed TRL version.

Fix: load tokenizer with AutoTokenizer, wrap its apply_chat_template so
every invocation passes enable_thinking=False, then hand the patched
tokenizer to GRPOTrainer via processing_class (falling back to the older
tokenizer kwarg when needed). A startup probe logs whether <think>
tokens still appear in the generation prompt.

Made-with: Cursor

training/train_grpo.py

@@ -12,7 +12,7 @@ from pathlib import Path
 from typing import Literal
 
 from datasets import load_dataset
-from transformers import TrainerCallback
+from transformers import AutoTokenizer, TrainerCallback
 from trl import GRPOConfig, GRPOTrainer
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
@@ -434,6 +434,43 @@ class LiveLogHistoryCallback(TrainerCallback):
         self._write(state.log_history)
 
 
+def _build_tokenizer_without_thinking(model_name: str):
+    """
+    Load the tokenizer and force Qwen3's chat template to skip the <think> block.
+
+    Qwen3 chat templates default to enable_thinking=True, which prefixes every
+    assistant turn with '<think>\\n</think>\\n\\n' and causes the model to burn
+    the entire completion budget on reasoning text without emitting a tool call.
+    TRL's GRPOConfig on older versions silently drops chat_template_kwargs, so
+    we enforce the flag by wrapping apply_chat_template at the tokenizer level.
+    """
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    original_apply = tokenizer.apply_chat_template
+
+    def apply_no_thinking(*args, **kwargs):
+        kwargs.setdefault("enable_thinking", False)
+        return original_apply(*args, **kwargs)
+
+    try:
+        tokenizer.apply_chat_template = apply_no_thinking
+        probe = tokenizer.apply_chat_template(
+            [{"role": "user", "content": "ping"}],
+            tokenize=False,
+            add_generation_prompt=True,
+        )
+        if "<think>" in probe:
+            print(
+                "Warning: chat template still emits <think> tokens; thinking mode may be on.",
+                flush=True,
+            )
+        else:
+            print("Chat template confirmed with enable_thinking=False.", flush=True)
+    except Exception as exc:
+        print(f"Chat template probe failed: {exc}", flush=True)
+
+    return tokenizer
+
+
 def summarize_log_history(log_history):
     reward_rows = [row for row in log_history if "reward" in row]
     if not reward_rows:
@@ -481,11 +518,14 @@ def main():
     dataset = load_dataset("json", data_files={"train": args.train_file})
     training_args = build_config(args)
 
+    tokenizer = _build_tokenizer_without_thinking(args.model_name)
+
     print("Initializing GRPO Trainer...", flush=True)
     trainer_kwargs = {
         "model": args.model_name,
         "args": training_args,
         "train_dataset": dataset["train"],
+        "processing_class": tokenizer,
     }
 
     if supports_env_factory:
@@ -499,6 +539,12 @@ def main():
         )
         trainer_kwargs["reward_funcs"] = [compatibility_reward_func]
 
+    trainer_params = inspect.signature(GRPOTrainer.__init__).parameters
+    if "processing_class" not in trainer_params:
+        trainer_kwargs.pop("processing_class", None)
+        if "tokenizer" in trainer_params:
+            trainer_kwargs["tokenizer"] = tokenizer
+
     trainer = GRPOTrainer(**trainer_kwargs)
     trainer.add_callback(LiveLogHistoryCallback(args.metrics_json))