Download fengshen/data/bert_dataloader/auto_split.sh from fclong/summary: direct link, hf CLI and curl.
- Browser
- Download file 230 Bytes
-
https://huggingface.co/spaces/fclong/summary/resolve/main/fengshen/data/bert_dataloader/auto_split.sh
- Command line
-
hf download hf://spaces/fclong/summary/fengshen/data/bert_dataloader/auto_split.sh
-
curl -L -o auto_split.sh https://huggingface.co/spaces/fclong/summary/resolve/main/fengshen/data/bert_dataloader/auto_split.sh
230 Bytes
| files=`find $1 -type f -size +1024M` | |
| for p in $files | |
| do | |
| echo "processing $p" | |
| name=`basename $p .json` | |
| file=`dirname $p` | |
| split -a 2 -C 300M $p $file/$name- && ls|grep -E "(-[a-zA-Z]{2})" |xargs -n1 -i{} mv {} {}.json | |
| rm -f $p | |
| done |