File size: 469 Bytes
f338eb3 | 1 2 3 4 5 6 7 8 9 10 11 | WORK_DIR=your/work/dir
LOCAL_DIR=$WORK_DIR/data/nq_hotpotqa_train
## process multiple dataset search format train file
DATA=nq,hotpotqa
python $WORK_DIR/scripts/data_process/qa_search_train_merge.py --local_dir $LOCAL_DIR --data_sources $DATA
## process multiple dataset search format test file
DATA=nq,triviaqa,popqa,hotpotqa,2wikimultihopqa,musique,bamboogle
python $WORK_DIR/scripts/data_process/qa_search_test_merge.py --local_dir $LOCAL_DIR --data_sources $DATA
|