update

examples/tutorials/dpo/ultrachat/step_2_train_sft_model_deepspeed.py

@@ -130,6 +130,11 @@ def filter_func(example, tokenizer, response_template):
     input_ids = example["input_ids"]
     text = tokenizer.decode(input_ids, skip_special_tokens=False)
 
+    max_length = 1024 if debug_mode else 4096,
+    length = len(input_ids)
+    if length > max_length:
+        raise AssertionError()
+
     if text.__contains__(response_template):
         result = True
     else: