File size: 2,526 Bytes
932bc69 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 | #!/usr/bin/env bash
# Source inside the container before data preparation, training or evaluation.
REPO="${REPO:-$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd)}"
export REPO
export ROOT="${ROOT:-$(dirname -- "$REPO")}"
export MODEL="${MODEL:-$ROOT/model_weights/Qwen--Qwen3.6-35B-A3B}"
export SPEC_PYTHON="${SPEC_PYTHON:-/opt/speculators_venv/bin/python}"
export TORCHRUN="${TORCHRUN:-/opt/speculators_venv/bin/torchrun}"
export VLLM_PYTHON="${VLLM_PYTHON:-/opt/vllm_venv/bin/python}"
export PYTHONPATH="$REPO/src:$REPO/hs_connectors/src${PYTHONPATH:+:$PYTHONPATH}"
export HF_ENDPOINT=https://huggingface.co
export HF_HOME="$ROOT/.cache/huggingface"
export HF_DATASETS_CACHE="$ROOT/.cache/huggingface/datasets"
export XDG_CACHE_HOME="$ROOT/.cache"
export TORCH_HOME="$ROOT/.cache/torch"
export VLLM_PLUGINS=""
export PYTHONUNBUFFERED=1
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-8}"
export MKL_NUM_THREADS="${MKL_NUM_THREADS:-8}"
export TOKENIZERS_PARALLELISM=false
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
export VLLM_NUM_GPUS="${VLLM_NUM_GPUS:-1}"
export WANDB_PROJECT="${WANDB_PROJECT:-speculators-ngram-candidates-validation}"
export WANDB_MODE="${WANDB_MODE:-online}"
export WANDB_CACHE_DIR="$ROOT/.cache/wandb"
export WANDB_CONFIG_DIR="$ROOT/.cache/wandb-config"
export WANDB_INIT_TIMEOUT=120
export NO_PROXY="${NO_PROXY:+$NO_PROXY,}127.0.0.1,localhost"
export no_proxy="$NO_PROXY"
export RUN_NAME="${RUN_NAME:-ngram-candidates-resume-validation}"
export LOG_DIR="${LOG_DIR:-$REPO/logs/$RUN_NAME}"
export WANDB_DIR="$LOG_DIR"
export DATA_DIR="${DATA_DIR:-$ROOT/datasets/qwen3.6-35b-a3b/resume_validation4096}"
export OUTPUT_DIR="${OUTPUT_DIR:-$ROOT/model_weights/ngram_candidates_qwen3_6-35b-a3b-perfectblend}"
mkdir -p "$LOG_DIR"
if [[ "$WANDB_MODE" != disabled ]]; then
# This workflow explicitly uses ROOT/cred.txt. A stale key inherited from
# the controller/login environment must not override a user-updated file.
WANDB_API_KEY="$("$SPEC_PYTHON" - "$ROOT/cred.txt" <<'PY'
import re
import sys
from pathlib import Path
keys = []
for line in Path(sys.argv[1]).read_text().splitlines():
value = line.strip()
if not value or value.startswith('#') or re.search(r'github_pat_|gh[pousr]_', value):
continue
keys.append(re.sub(r'^[A-Za-z_][A-Za-z0-9_ -]*\s*[:=]\s*', '', value).strip("\"'"))
if len(keys) != 1:
raise SystemExit('Expected exactly one W&B key in the runtime credential file')
print(keys[0])
PY
)" || return 1
export WANDB_API_KEY
fi
|