source/scripts/cluster/ngram_prepare_data.sh

#!/usr/bin/env bash
set -euo pipefail
export WANDB_MODE=disabled
source "$(dirname -- "${BASH_SOURCE[0]}")/ngram_runtime_env.sh"
cd "$REPO"
PREPARED_DATA="$ROOT/datasets/qwen3.6-35b-a3b/qwen3.6-35b-a3b_train_spec_len3072_fullvocab"
VALIDATION_DATA="$ROOT/datasets/qwen3.6-35b-a3b/resume_validation4096"
"$SPEC_PYTHON" scripts/prepare_data.py \
    --model "$MODEL" \
    --data "$ROOT/datasets/spec-regenerated-datasets/qwen3.6-35b-a3b/train" \
    --output "$VALIDATION_DATA" \
    --seq-length 3072 \
    --max-samples 4096 \
    --num-preprocessing-workers 8
"$SPEC_PYTHON" scripts/prepare_data.py \
    --model "$MODEL" \
    --data "$ROOT/datasets/spec-regenerated-datasets/qwen3.6-35b-a3b/train" \
    --output "$PREPARED_DATA" \
    --seq-length 3072 \
    --num-preprocessing-workers 16
"$SPEC_PYTHON" - "$PREPARED_DATA" "$VALIDATION_DATA" <<'PY'
import json
import sys
from pathlib import Path

from datasets import load_from_disk

source, target = map(Path, sys.argv[1:])
data = load_from_disk(str(source))
summary = {'full_rows': len(data), 'validation_rows': len(load_from_disk(str(target))), 'max_length': 3072}
Path('logs/ngram_setup/preparation-summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(summary)
PY