alternative dataset load

scripts/train.py

@@ -33,7 +33,9 @@ def main():
     tokenizer.pad_token = tokenizer.eos_token
     
     # Load dataset
-    dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train", keep_in_memory=True)
+    os.environ["HF_DATASETS_CACHE"] = "/tmp/hf-datasets-cache"  # or any valid path
+    dataset = load_dataset("allenai/tulu-v2-sft-mixture", split="train", download_mode="force_redownload")
+
 
     def tokenize_function(examples):
         return tokenizer(