PLI Labs public snapshot 2026-10-28 (history squashed)

0693474 6 days ago

2.77 kB

#!/usr/bin/env bash
# DSV41-OFFLOAD maintenance window on the Sparks (run on sparkhead from ~/dsv41-deploy).
#   offload/launch/window.sh open "<purpose>"   wait until the serve is idle, write HOLD (pauses the local-law K24 queue),
#                                               start a placeholder `vllm serve [DSV41-OFFLOAD hold]` process on both nodes
#                                               (the NVO drama3 waiter treats a node with a `vllm serve` process as busy)
#   offload/launch/window.sh close             production must answer on :8000 with /srv/dsv41/legal-v2a loaded; then
#                                               remove the placeholders and HOLD
#   offload/launch/window.sh status
set -euo pipefail; cd "$(dirname "$0")/../.."; set -a; . ./dsv41.env; set +a
HOLDF=$HOME/dsv41-deploy/HOLD   # the production checkout; the K24 driver reads exactly this path
# idle: the metrics fetch must succeed (HTTP 2xx) and BOTH request gauges must be present and zero; anything else is busy
# (S5R fix 3: an empty or partial page used to read as idle)
idle() {
  local m; m=$(curl -sf -m 5 http://localhost:8000/metrics) || return 1
  printf '%s\n' "$m" | awk '/^vllm:num_requests_running\{/{r=1; s+=$NF} /^vllm:num_requests_waiting\{/{w=1; s+=$NF}
    END{exit !(r && w && s==0)}'
}
holdproc() { ssh -o BatchMode=yes "$1" "tmux has-session -t dsv41-hold 2>/dev/null || tmux new-session -d -s dsv41-hold 'exec -a \"vllm serve [DSV41-OFFLOAD hold]\" sleep 86400'"; }
case "${1:-status}" in
  open)
    purpose=${2:?purpose}
    # a running serve must be drained first; a stopped one is idle by definition
    if curl -sf -m 5 http://localhost:8000/v1/models >/dev/null; then
      for i in $(seq 1 240); do idle && break; sleep 15; done
      idle || { echo "serve still busy after 60 min; not opening" >&2; exit 1; }
    fi
    echo "$(date -u +%FT%TZ) DSV41-OFFLOAD (session coordinator): $purpose" > "$HOME/dsv41-deploy/HOLD"
    for n in $HEAD $WORKER; do holdproc $n; done
    echo "window open: $(cat "$HOME/dsv41-deploy/HOLD")" ;;
  close)
    root=$(curl -s -m 5 http://localhost:8000/v1/models | python3 -c 'import sys,json; print(json.load(sys.stdin)["data"][0]["root"])' 2>/dev/null || true)
    [ "$root" = /srv/dsv41/legal-v2a ] || { echo "production not up on :8000 with legal-v2a (root='$root'); window stays open" >&2; exit 1; }
    for n in $HEAD $WORKER; do ssh -o BatchMode=yes $n "tmux kill-session -t dsv41-hold 2>/dev/null || true"; done
    rm -f "$HOME/dsv41-deploy/HOLD"; echo "window closed" ;;
  status)
    [ -f "$HOME/dsv41-deploy/HOLD" ] && echo "HOLD: $(cat "$HOME/dsv41-deploy/HOLD")" || echo "no HOLD"
    for n in $HEAD $WORKER; do echo "$n: $(ssh $n 'pgrep -af "vllm serve" | cut -c1-90 | tr "\n" ";"')"; done ;;
esac

	#!/usr/bin/env bash
	# DSV41-OFFLOAD maintenance window on the Sparks (run on sparkhead from ~/dsv41-deploy).
	# offload/launch/window.sh open "<purpose>" wait until the serve is idle, write HOLD (pauses the local-law K24 queue),
	# start a placeholder `vllm serve [DSV41-OFFLOAD hold]` process on both nodes
	# (the NVO drama3 waiter treats a node with a `vllm serve` process as busy)
	# offload/launch/window.sh close production must answer on :8000 with /srv/dsv41/legal-v2a loaded; then
	# remove the placeholders and HOLD
	# offload/launch/window.sh status
	set -euo pipefail; cd "$(dirname "$0")/../.."; set -a; . ./dsv41.env; set +a
	HOLDF=$HOME/dsv41-deploy/HOLD # the production checkout; the K24 driver reads exactly this path
	# idle: the metrics fetch must succeed (HTTP 2xx) and BOTH request gauges must be present and zero; anything else is busy
	# (S5R fix 3: an empty or partial page used to read as idle)
	idle() {
	local m; m=$(curl -sf -m 5 http://localhost:8000/metrics) \|\| return 1
	printf '%s\n' "$m" \| awk '/^vllm:num_requests_running\{/{r=1; s+=$NF} /^vllm:num_requests_waiting\{/{w=1; s+=$NF}
	END{exit !(r && w && s==0)}'
	}
	holdproc() { ssh -o BatchMode=yes "$1" "tmux has-session -t dsv41-hold 2>/dev/null \|\| tmux new-session -d -s dsv41-hold 'exec -a \"vllm serve [DSV41-OFFLOAD hold]\" sleep 86400'"; }
	case "${1:-status}" in
	open)
	purpose=${2:?purpose}
	# a running serve must be drained first; a stopped one is idle by definition
	if curl -sf -m 5 http://localhost:8000/v1/models >/dev/null; then
	for i in $(seq 1 240); do idle && break; sleep 15; done
	idle \|\| { echo "serve still busy after 60 min; not opening" >&2; exit 1; }
	fi
	echo "$(date -u +%FT%TZ) DSV41-OFFLOAD (session coordinator): $purpose" > "$HOME/dsv41-deploy/HOLD"
	for n in $HEAD $WORKER; do holdproc $n; done
	echo "window open: $(cat "$HOME/dsv41-deploy/HOLD")" ;;
	close)
	root=$(curl -s -m 5 http://localhost:8000/v1/models \| python3 -c 'import sys,json; print(json.load(sys.stdin)["data"][0]["root"])' 2>/dev/null \|\| true)
	[ "$root" = /srv/dsv41/legal-v2a ] \|\| { echo "production not up on :8000 with legal-v2a (root='$root'); window stays open" >&2; exit 1; }
	for n in $HEAD $WORKER; do ssh -o BatchMode=yes $n "tmux kill-session -t dsv41-hold 2>/dev/null \|\| true"; done
	rm -f "$HOME/dsv41-deploy/HOLD"; echo "window closed" ;;
	status)
	[ -f "$HOME/dsv41-deploy/HOLD" ] && echo "HOLD: $(cat "$HOME/dsv41-deploy/HOLD")" \|\| echo "no HOLD"
	for n in $HEAD $WORKER; do echo "$n: $(ssh $n 'pgrep -af "vllm serve" \| cut -c1-90 \| tr "\n" ";"')"; done ;;
	esac