| #SBATCH --job-name=ar-dataval | |
| #SBATCH --partition=dev | |
| #SBATCH --nodes=1 | |
| #SBATCH --ntasks-per-node=1 | |
| #SBATCH --cpus-per-task=32 | |
| #SBATCH --output=dataval_%j.log | |
| set -euo pipefail | |
| cd /fsx/ajay-stu/AR | |
| source .venv/bin/activate | |
| OUT=/fsx/ajay-stu/ar-data/bin-val | |
| echo "=== fineweb ===" | |
| python -m ar.data.prepare_data --source fineweb --target_tokens 1000000000 --out_dir $OUT | |
| echo "=== stack ===" | |
| python -m ar.data.prepare_data --source stack --target_tokens 1000000000 --out_dir $OUT | |
| echo "=== finemath ===" | |
| python -m ar.data.prepare_data --source finemath --target_tokens 1000000000 --out_dir $OUT | |
| echo "=== sizes ===" | |
| ls -la $OUT |