PULSE-code / slurm /verify_data.sbatch
velvet-pine-22's picture
PULSE-code: benchmark code for the PULSE dataset (camera-ready release)
dffbc1f verified
Raw History Blame Contribute Delete
903 Bytes
#!/bin/bash
# sha256 / git-blob check of a dataset copy against the Hugging Face v1.1 file list.
# mkdir -p runs/_jobs && PULSE_ROOT=/path/to/PULSE PULSE_PYTHON=/path/to/python sbatch slurm/verify_data.sbatch
#SBATCH --job-name=pulse-verify
#SBATCH --partition=cpuXeon6458
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=16
#SBATCH --mem=16G
#SBATCH --time=02:00:00
#SBATCH --output=runs/_jobs/%x_%j.out
set -euo pipefail
ulimit -n 51200
cd "${SLURM_SUBMIT_DIR}"
: "${PULSE_ROOT:?set PULSE_ROOT to the dataset root}"
PY="${PULSE_PYTHON:-python}"
"$PY" scripts/verify_data.py --root "$PULSE_ROOT" \
--manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 \
--expect-different README.md --expect-different croissant.json \
--ignore-extra 'metadata/annotations/logs/*' --ignore-extra 'metadata/annotations/_run_summary.json' \
--out "runs/_jobs/verify_report_${SLURM_JOB_ID}.json"