File size: 1,242 Bytes
932bc69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | #!/usr/bin/env bash
set -euo pipefail
export WANDB_MODE=disabled
source "$(dirname -- "${BASH_SOURCE[0]}")/ngram_runtime_env.sh"
cd "$REPO"
PREPARED_DATA="$ROOT/datasets/qwen3.6-35b-a3b/qwen3.6-35b-a3b_train_spec_len3072_fullvocab"
VALIDATION_DATA="$ROOT/datasets/qwen3.6-35b-a3b/resume_validation4096"
"$SPEC_PYTHON" scripts/prepare_data.py \
--model "$MODEL" \
--data "$ROOT/datasets/spec-regenerated-datasets/qwen3.6-35b-a3b/train" \
--output "$VALIDATION_DATA" \
--seq-length 3072 \
--max-samples 4096 \
--num-preprocessing-workers 8
"$SPEC_PYTHON" scripts/prepare_data.py \
--model "$MODEL" \
--data "$ROOT/datasets/spec-regenerated-datasets/qwen3.6-35b-a3b/train" \
--output "$PREPARED_DATA" \
--seq-length 3072 \
--num-preprocessing-workers 16
"$SPEC_PYTHON" - "$PREPARED_DATA" "$VALIDATION_DATA" <<'PY'
import json
import sys
from pathlib import Path
from datasets import load_from_disk
source, target = map(Path, sys.argv[1:])
data = load_from_disk(str(source))
summary = {'full_rows': len(data), 'validation_rows': len(load_from_disk(str(target))), 'max_length': 3072}
Path('logs/ngram_setup/preparation-summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(summary)
PY
|