diffusion_baseline / upstreams /flm /configs /data /openwebtext-split-reflow.yaml

Add files using upload-large-folder tool

93cd35a verified 5 days ago

215 Bytes

train: reflow-dataset
valid: openwebtext-valid
tokenizer_name_or_path: gpt2
cache_dir: /PATH/TO/TRAIN/DATASET
wrap: True
streaming: False
insert_train_eos: True
insert_valid_eos: True
save_dir: /PATH/TO/SAVE/DATASET

	train: reflow-dataset
	valid: openwebtext-valid
	tokenizer_name_or_path: gpt2
	cache_dir: /PATH/TO/TRAIN/DATASET
	wrap: True
	streaming: False
	insert_train_eos: True
	insert_valid_eos: True
	save_dir: /PATH/TO/SAVE/DATASET