#!/bin/bash
#SBATCH --job-name=ar-dataval
#SBATCH --partition=dev
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=32
#SBATCH --output=dataval_%j.log

set -euo pipefail

cd /fsx/ajay-stu/AR
source .venv/bin/activate

OUT=/fsx/ajay-stu/ar-data/bin-val

echo "=== fineweb ==="
python -m ar.data.prepare_data --source fineweb  --target_tokens 1000000000 --out_dir $OUT
echo "=== stack ==="
python -m ar.data.prepare_data --source stack    --target_tokens 1000000000 --out_dir $OUT
echo "=== finemath ==="
python -m ar.data.prepare_data --source finemath --target_tokens 1000000000 --out_dir $OUT

echo "=== sizes ==="
ls -la $OUT