23f2002275 commited on
Commit ·
eae16b1
1
Parent(s): d2b6a7e
fix(job_train): pip install huggingface_hub before downloading data files (R1 parity with job_smoke.sh)
Browse files- scripts/job_train.sh +3 -0
scripts/job_train.sh
CHANGED
|
@@ -5,6 +5,9 @@ set -e
|
|
| 5 |
cd /w
|
| 6 |
export PYTHONPATH="/w:${PYTHONPATH}"
|
| 7 |
|
|
|
|
|
|
|
|
|
|
| 8 |
# Fetch large data files from HF (uploaded separately via hf upload)
|
| 9 |
mkdir -p data
|
| 10 |
python -c "from huggingface_hub import hf_hub_download; [hf_hub_download(repo_id='Pratham-math/fathom-code', filename=f'data/{f}', local_dir='/w', token='${HF_TOKEN}') for f in ['train.jsonl','eval.jsonl','sft_traces.jsonl']]"
|
|
|
|
| 5 |
cd /w
|
| 6 |
export PYTHONPATH="/w:${PYTHONPATH}"
|
| 7 |
|
| 8 |
+
# Install huggingface_hub first so we can download data files (parity with job_smoke.sh)
|
| 9 |
+
pip install -q 'huggingface_hub>=0.28'
|
| 10 |
+
|
| 11 |
# Fetch large data files from HF (uploaded separately via hf upload)
|
| 12 |
mkdir -p data
|
| 13 |
python -c "from huggingface_hub import hf_hub_download; [hf_hub_download(repo_id='Pratham-math/fathom-code', filename=f'data/{f}', local_dir='/w', token='${HF_TOKEN}') for f in ['train.jsonl','eval.jsonl','sft_traces.jsonl']]"
|