| #!/usr/bin/env bash |
| set -euo pipefail |
| SIMCT_BASE=/workspace/storage-shared/nlp/tungks |
| SIMCT_HOST=tungdd11-sparse-vllm-core-0-0 |
| [[ "$(hostname)" == "$SIMCT_HOST" ]] || { echo 'WRONG_HOST: run only on the newly borrowed node'; exit 2; } |
| SIMCT_POINTER="$SIMCT_BASE/borrow8-$SIMCT_HOST.launch" |
| if [[ -e "$SIMCT_POINTER" ]]; then |
| echo 'ALLOCATION_ALREADY_RECORDED: inspect/recover this allocation; do not reset its clock' |
| cat "$SIMCT_POINTER" |
| exit 2 |
| fi |
| SIMCT_STARTED="${SIMCT_BORROW_STARTED:-$(date +%s)}" |
| [[ "$SIMCT_STARTED" =~ ^[0-9]+$ ]] || { echo 'START must be Unix integer seconds'; exit 2; } |
| SIMCT_TASK="$(mktemp -d "$SIMCT_BASE/borrow8-XXXXXXXX")" |
| (set -o noclobber; printf '%s\n%s\n' "$SIMCT_TASK" "$SIMCT_STARTED" > "$SIMCT_POINTER") |
| printf 'ALLOCATION_ROOT=%s\nSTARTED=%s\n' "$SIMCT_TASK" "$SIMCT_STARTED" |
| test -x /usr/bin/python3.12 || { echo 'MISSING_RUNTIME: /usr/bin/python3.12; keep the recorded allocation clock'; exit 2; } |
| test -x /opt/venvs/simct-b200/bin/python || { echo 'MISSING_RUNTIME: /opt/venvs/simct-b200/bin/python; keep the recorded allocation clock'; exit 2; } |
| SIMCT_REV=d849970059510104e763209685b80645af5f88d9 |
| SIMCT_URL="https://huggingface.co/codemaivanngu/simct/resolve/$SIMCT_REV" |
| for SIMCT_FILE in simct-b200-portable.bundle job-manager-source.tar; do |
| curl --fail --location --silent --show-error --retry 3 --connect-timeout 20 \ |
| --proxy http://10.30.154.118:80 --noproxy "" \ |
| "$SIMCT_URL/$SIMCT_FILE" -o "$SIMCT_TASK/$SIMCT_FILE" |
| done |
| ( |
| cd "$SIMCT_TASK" |
| printf '%s %s\n' \ |
| 885f2fb7d994720f2a06e1a151493bb2b8c3ff23c8aaff1002202b02102df6a3 simct-b200-portable.bundle \ |
| cdbda29a2d35fef8c3315a81ba4bf6d1cb7402b762cf8609379d0e60773c4914 job-manager-source.tar \ |
| | sha256sum --check - |
| ) |
| git clone --branch vdt/ops/b200-portable "$SIMCT_TASK/simct-b200-portable.bundle" "$SIMCT_TASK/source" |
| [[ "$(git -C "$SIMCT_TASK/source" rev-parse HEAD)" == 6b9442ad463dc3781436c7cc94bb4e49d49c81d8 ]] || { echo 'SOURCE_MISMATCH'; exit 2; } |
| /usr/bin/python3.12 - "$SIMCT_TASK" <<'PY' |
| import sys,tarfile |
| from pathlib import Path |
| root=Path(sys.argv[1]) |
| with tarfile.open(root/'job-manager-source.tar') as archive: |
| for member in archive.getmembers(): |
| if member.name!='job-manager' and not member.name.startswith('job-manager/'): |
| raise ValueError('Unexpected archive member') |
| archive.extractall(root,filter='data') |
| PY |
| bash "$SIMCT_TASK/source/experiments/runai/start_borrowed_campaign.sh" \ |
| "$SIMCT_TASK/work" "$SIMCT_TASK/job-manager" "$SIMCT_STARTED" "$SIMCT_HOST" |
|
|
