File size: 2,538 Bytes
1a61f92 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | #!/usr/bin/env bash
set -euo pipefail
SIMCT_BASE=/workspace/storage-shared/nlp/tungks
SIMCT_HOST=tungdd11-sparse-vllm-core-0-0
[[ "$(hostname)" == "$SIMCT_HOST" ]] || { echo 'WRONG_HOST: run only on the newly borrowed node'; exit 2; }
SIMCT_POINTER="$SIMCT_BASE/borrow8-$SIMCT_HOST.launch"
if [[ -e "$SIMCT_POINTER" ]]; then
echo 'ALLOCATION_ALREADY_RECORDED: inspect/recover this allocation; do not reset its clock'
cat "$SIMCT_POINTER"
exit 2
fi
SIMCT_STARTED="${SIMCT_BORROW_STARTED:-$(date +%s)}"
[[ "$SIMCT_STARTED" =~ ^[0-9]+$ ]] || { echo 'START must be Unix integer seconds'; exit 2; }
SIMCT_TASK="$(mktemp -d "$SIMCT_BASE/borrow8-XXXXXXXX")"
(set -o noclobber; printf '%s\n%s\n' "$SIMCT_TASK" "$SIMCT_STARTED" > "$SIMCT_POINTER")
printf 'ALLOCATION_ROOT=%s\nSTARTED=%s\n' "$SIMCT_TASK" "$SIMCT_STARTED"
test -x /usr/bin/python3.12 || { echo 'MISSING_RUNTIME: /usr/bin/python3.12; keep the recorded allocation clock'; exit 2; }
test -x /opt/venvs/simct-b200/bin/python || { echo 'MISSING_RUNTIME: /opt/venvs/simct-b200/bin/python; keep the recorded allocation clock'; exit 2; }
SIMCT_REV=d849970059510104e763209685b80645af5f88d9
SIMCT_URL="https://huggingface.co/codemaivanngu/simct/resolve/$SIMCT_REV"
for SIMCT_FILE in simct-b200-portable.bundle job-manager-source.tar; do
curl --fail --location --silent --show-error --retry 3 --connect-timeout 20 \
--proxy http://10.30.154.118:80 --noproxy "" \
"$SIMCT_URL/$SIMCT_FILE" -o "$SIMCT_TASK/$SIMCT_FILE"
done
(
cd "$SIMCT_TASK"
printf '%s %s\n' \
885f2fb7d994720f2a06e1a151493bb2b8c3ff23c8aaff1002202b02102df6a3 simct-b200-portable.bundle \
cdbda29a2d35fef8c3315a81ba4bf6d1cb7402b762cf8609379d0e60773c4914 job-manager-source.tar \
| sha256sum --check -
)
git clone --branch vdt/ops/b200-portable "$SIMCT_TASK/simct-b200-portable.bundle" "$SIMCT_TASK/source"
[[ "$(git -C "$SIMCT_TASK/source" rev-parse HEAD)" == 6b9442ad463dc3781436c7cc94bb4e49d49c81d8 ]] || { echo 'SOURCE_MISMATCH'; exit 2; }
/usr/bin/python3.12 - "$SIMCT_TASK" <<'PY'
import sys,tarfile
from pathlib import Path
root=Path(sys.argv[1])
with tarfile.open(root/'job-manager-source.tar') as archive:
for member in archive.getmembers():
if member.name!='job-manager' and not member.name.startswith('job-manager/'):
raise ValueError('Unexpected archive member')
archive.extractall(root,filter='data')
PY
bash "$SIMCT_TASK/source/experiments/runai/start_borrowed_campaign.sh" \
"$SIMCT_TASK/work" "$SIMCT_TASK/job-manager" "$SIMCT_STARTED" "$SIMCT_HOST"
|