Diagnose Qwen generation termination

app.py

@@ -597,7 +597,7 @@ def _load_qwen38():
         return processor, model
 
 
-def _generate_qwen38(prompt, max_new_tokens=64, temperature=0.7, thinking=False):
+def _generate_qwen38(prompt, max_new_tokens=256, temperature=0.7, thinking=True):
     processor, model = _load_qwen38()
     max_new_tokens = max(1, min(256, int(max_new_tokens)))
     temperature = max(0.0, min(2.0, float(temperature)))
@@ -614,6 +614,10 @@ def _generate_qwen38(prompt, max_new_tokens=64, temperature=0.7, thinking=False)
         text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
     inputs = processor(text=[text], return_tensors="pt")
     inputs = {key: value.to("cpu") if hasattr(value, "to") else value for key, value in inputs.items()}
+    tokenizer = processor.tokenizer
+    eos_id = tokenizer.eos_token_id
+    pad_id = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else eos_id
+    input_tokens = int(inputs["input_ids"].shape[-1])
     t0 = time.time()
     with torch.inference_mode():
         output = model.generate(
@@ -621,12 +625,38 @@ def _generate_qwen38(prompt, max_new_tokens=64, temperature=0.7, thinking=False)
             max_new_tokens=max_new_tokens,
             do_sample=temperature > 0,
             temperature=max(temperature, 0.01),
+            eos_token_id=eos_id,
+            pad_token_id=pad_id,
+            return_dict_in_generate=True,
+            output_scores=True,
         )
     elapsed = max(time.time() - t0, 0.001)
-    input_tokens = int(inputs["input_ids"].shape[-1])
-    generated = output[0, input_tokens:]
-    answer = processor.tokenizer.decode(generated, skip_special_tokens=True).strip()
-    return {"text": answer, "prompt_tokens": input_tokens, "completion_tokens": int(generated.shape[-1]), "generation_tokens_per_sec": round(float(generated.shape[-1]) / elapsed, 2), "inference_time_sec": round(elapsed, 3), "device": "cpu", "dtype": "bfloat16", "thinking": bool(thinking)}
+    sequences = output.sequences
+    generated = sequences[0, input_tokens:]
+    eos_positions = (generated == eos_id).nonzero(as_tuple=True)[0].tolist() if eos_id is not None else []
+    answer = tokenizer.decode(generated, skip_special_tokens=True).strip()
+    return {
+        "text": answer,
+        "prompt_tokens": input_tokens,
+        "completion_tokens": int(generated.shape[-1]),
+        "generation_tokens_per_sec": round(float(generated.shape[-1]) / elapsed, 2),
+        "inference_time_sec": round(elapsed, 3),
+        "device": "cpu",
+        "dtype": "bfloat16",
+        "thinking": bool(thinking),
+        "debug": {
+            "rendered_chat_template": text,
+            "input_ids": inputs["input_ids"][0].tolist(),
+            "generated_token_ids": generated.tolist(),
+            "eos_token_id": eos_id,
+            "pad_token_id": pad_id,
+            "eos_positions": eos_positions,
+            "sequence_length": int(sequences.shape[-1]),
+            "max_new_tokens": max_new_tokens,
+            "scores_steps": len(output.scores) if output.scores is not None else 0,
+            "finish_reason": "eos" if eos_positions else ("length" if generated.shape[-1] >= max_new_tokens else "unknown"),
+        },
+    }
 
 
 def _load_tinyllama():