File size: 647 Bytes
2ae6691 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | #!/bin/bash
#SBATCH --job-name=ar-dataval
#SBATCH --partition=dev
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=32
#SBATCH --output=dataval_%j.log
set -euo pipefail
cd /fsx/ajay-stu/AR
source .venv/bin/activate
OUT=/fsx/ajay-stu/ar-data/bin-val
echo "=== fineweb ==="
python -m ar.data.prepare_data --source fineweb --target_tokens 1000000000 --out_dir $OUT
echo "=== stack ==="
python -m ar.data.prepare_data --source stack --target_tokens 1000000000 --out_dir $OUT
echo "=== finemath ==="
python -m ar.data.prepare_data --source finemath --target_tokens 1000000000 --out_dir $OUT
echo "=== sizes ==="
ls -la $OUT |