key value

scripts/train.py

@@ -37,8 +37,9 @@ def main():
     dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train")
     
     def tokenize_function(examples):
+        text_key = "content" if "content" in examples else "text"
         return tokenizer(
-            examples["text"],
+            examples[text_key],
             truncation=True,
             max_length=4096,
             padding="max_length"