#!/bin/bash # sha256 / git-blob check of a dataset copy against the Hugging Face v1.1 file list. # mkdir -p runs/_jobs && PULSE_ROOT=/path/to/PULSE PULSE_PYTHON=/path/to/python sbatch slurm/verify_data.sbatch #SBATCH --job-name=pulse-verify #SBATCH --partition=cpuXeon6458 #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=16 #SBATCH --mem=16G #SBATCH --time=02:00:00 #SBATCH --output=runs/_jobs/%x_%j.out set -euo pipefail ulimit -n 51200 cd "${SLURM_SUBMIT_DIR}" : "${PULSE_ROOT:?set PULSE_ROOT to the dataset root}" PY="${PULSE_PYTHON:-python}" "$PY" scripts/verify_data.py --root "$PULSE_ROOT" \ --manifest manifests/hf_PULSE_v1.1_files.tsv --workers 16 \ --expect-different README.md --expect-different croissant.json \ --ignore-extra 'metadata/annotations/logs/*' --ignore-extra 'metadata/annotations/_run_summary.json' \ --out "runs/_jobs/verify_report_${SLURM_JOB_ID}.json"