| #!/usr/bin/env bash |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
|
|
| set -Eeuo pipefail |
|
|
| readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" |
| readonly REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)" |
|
|
| readonly RUN_ALL="${SCRIPT_DIR}/run_all.sh" |
| readonly VOCAB_SCRIPT="${REPO_ROOT}/scripts/build_vocab_mapping.py" |
|
|
| PYTHON_BIN="${PARSER2_PYTHON_BIN:-${REPO_ROOT}/speculators_venv/bin/python}" |
| VLLM_BIN="${PARSER2_VLLM_BIN:-${REPO_ROOT}/vllm_venv/bin/vllm}" |
|
|
| SOURCE_JSONL="${PARSER2_SOURCE_JSONL:-/home/ma-user/work/data_mllm/new_datasets/swift_merged_datasets/version_v2.1/train_v2.1.jsonl}" |
| MODEL="${PARSER2_MODEL:-/home/ma-user/work/data_mllm/publish_models/Infinity-Parser2.1-Flash-2608}" |
| MEDIA_ROOT="${PARSER2_MEDIA_ROOT:-/inspire/sfs/project/inf-multimodal/public}" |
|
|
| |
| |
| DATA_ROOT="${PARSER2_DATA_ROOT:-/inspire/sfs/project/inf-multimodal/public/wumengke/datasets/infinity_parsers2_v2_1_max32768_vocab32k}" |
| OUTPUT_ROOT="${PARSER2_REGEN_ROOT:-${DATA_ROOT}/regen}" |
| FINAL_DIR="${PARSER2_FINAL_DIR:-${DATA_ROOT}/target_answers}" |
| PREPARED_ROOT="${PARSER2_PREPARED_ROOT:-${DATA_ROOT}/dflash_data}" |
|
|
| |
| |
| MAX_MODEL_LEN="${PARSER2_TEACHER_MAX_MODEL_LEN:-65536}" |
| MAX_TOKENS="${PARSER2_MAX_TOKENS:-16384}" |
| SEQ_LENGTH="${PARSER2_SEQ_LENGTH:-16384}" |
| DRAFT_VOCAB_SIZE="${PARSER2_DRAFT_VOCAB_SIZE:-248320}" |
| CONCURRENCY_PER_GPU="${PARSER2_CONCURRENCY_PER_GPU:-32}" |
| MAX_NUM_SEQS="${PARSER2_TEACHER_MAX_NUM_SEQS:-}" |
| MAX_IMAGES="${PARSER2_TEACHER_MAX_IMAGES:-16}" |
| API_HOST="${PARSER2_TEACHER_HOST:-0.0.0.0}" |
| API_PORT="${PARSER2_TEACHER_BASE_PORT:-8000}" |
| NODE_ADDRESS="${PARSER2_TEACHER_ADVERTISE_HOST:-}" |
| START_TIMEOUT="${PARSER2_TEACHER_START_TIMEOUT:-1800}" |
|
|
| NNODES="${PET_NNODES:-}" |
| NODE_RANK="${PET_NODE_RANK:-}" |
| LOCAL_DP_SIZE="${PET_NPROC_PER_NODE:-}" |
| GLOBAL_DP_SIZE="" |
| GLOBAL_CONCURRENCY="" |
| DIST_MASTER_ADDR="${MASTER_ADDR:-${PET_MASTER_ADDR:-}}" |
| DIST_MASTER_PORT="${MASTER_PORT:-${PET_MASTER_PORT:-}}" |
|
|
| declare -a VLLM_PIDS=() |
| declare -a ENDPOINTS=() |
| COORD_ACTIVE=0 |
| COORD_DIR="" |
| STOP_FILE="" |
| STARTED_FILE="" |
| ACK_FILE="" |
| FAILED_FILE="" |
| READY_FILE="" |
| LOG_DIR="" |
|
|
| die() { |
| echo "Error: $*" >&2 |
| exit 1 |
| } |
|
|
| is_positive_integer() { |
| [[ "$1" =~ ^[1-9][0-9]*$ ]] |
| } |
|
|
| require_file() { |
| [[ -f "$1" ]] || die "missing file: $1" |
| } |
|
|
| require_executable() { |
| [[ -x "$1" ]] || die "missing executable: $1" |
| } |
|
|
| validate_common_paths() { |
| require_executable "$PYTHON_BIN" |
| require_file "$RUN_ALL" |
| require_file "$VOCAB_SCRIPT" |
| require_file "$SOURCE_JSONL" |
| require_file "${MODEL}/config.json" |
| } |
|
|
| validate_cluster_topology() { |
| is_positive_integer "$NNODES" || die "PET_NNODES must be a positive integer" |
| [[ "$NODE_RANK" =~ ^[0-9]+$ ]] || die "PET_NODE_RANK must be a non-negative integer" |
| is_positive_integer "$LOCAL_DP_SIZE" || \ |
| die "PET_NPROC_PER_NODE must be a positive integer" |
| (( NNODES == 4 )) || die "this launcher requires PET_NNODES=4, got ${NNODES}" |
| (( NODE_RANK < NNODES )) || \ |
| die "PET_NODE_RANK must be in [0, $((NNODES - 1))], got ${NODE_RANK}" |
| [[ -n "$DIST_MASTER_ADDR" ]] || \ |
| die "MASTER_ADDR or PET_MASTER_ADDR must be set" |
| is_positive_integer "$DIST_MASTER_PORT" || \ |
| die "MASTER_PORT or PET_MASTER_PORT must be a positive integer" |
|
|
| GLOBAL_DP_SIZE=$((NNODES * LOCAL_DP_SIZE)) |
|
|
| is_positive_integer "$API_PORT" && (( API_PORT + LOCAL_DP_SIZE - 1 <= 65535 )) || \ |
| die "API ports ${API_PORT}..$((API_PORT + LOCAL_DP_SIZE - 1)) must be in [1, 65535]" |
|
|
| for value_name in MAX_MODEL_LEN MAX_TOKENS SEQ_LENGTH DRAFT_VOCAB_SIZE \ |
| CONCURRENCY_PER_GPU MAX_IMAGES START_TIMEOUT; do |
| is_positive_integer "${!value_name}" || \ |
| die "${value_name} must be a positive integer, got ${!value_name}" |
| done |
| if [[ -n "$MAX_NUM_SEQS" ]]; then |
| is_positive_integer "$MAX_NUM_SEQS" || \ |
| die "MAX_NUM_SEQS must be a positive integer, got ${MAX_NUM_SEQS}" |
| fi |
| GLOBAL_CONCURRENCY=$((GLOBAL_DP_SIZE * CONCURRENCY_PER_GPU)) |
| (( MAX_TOKENS < MAX_MODEL_LEN )) || \ |
| die "PARSER2_MAX_TOKENS=${MAX_TOKENS} leaves no prompt budget below max-model-len=${MAX_MODEL_LEN}; reduce output tokens or increase context length" |
| } |
|
|
| resolve_node_address() { |
| if [[ -z "$NODE_ADDRESS" ]]; then |
| |
| |
| NODE_ADDRESS="$("$PYTHON_BIN" - <<'PY' |
| import fcntl |
| import socket |
| import struct |
| |
| with open("/proc/net/route") as routes: |
| next(routes) # 表头 |
| defaults = [ |
| fields |
| for line in routes |
| if (fields := line.split())[1] == "00000000" |
| and fields[7] == "00000000" |
| and int(fields[3], 16) & 1 |
| ] |
| if not defaults: |
| raise SystemExit("no active default IPv4 route") |
| interface = min(defaults, key=lambda fields: int(fields[6]))[0] |
| |
| with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as sock: |
| address = fcntl.ioctl( |
| sock.fileno(), 0x8915, struct.pack("256s", interface.encode()) |
| ) # Linux SIOCGIFADDR |
| print(socket.inet_ntoa(address[20:24])) |
| PY |
| )" || die "cannot read local interface IPv4; set PARSER2_TEACHER_ADVERTISE_HOST to this node's reachable IP" |
| fi |
| echo "Node ${NODE_RANK}: advertising APIs on ${NODE_ADDRESS}" |
| } |
|
|
| configure_runtime_paths() { |
| local coord_tag |
| coord_tag="${DIST_MASTER_ADDR//[^[:alnum:]._-]/_}_${DIST_MASTER_PORT}" |
| COORD_DIR="${REPO_ROOT}/tmp/infinity_parser2_regeneration_4node_${coord_tag}" |
| STOP_FILE="${COORD_DIR}/stop" |
| STARTED_FILE="${COORD_DIR}/started.node${NODE_RANK}" |
| ACK_FILE="${COORD_DIR}/stopped.node${NODE_RANK}" |
| FAILED_FILE="${COORD_DIR}/failed.node${NODE_RANK}" |
| READY_FILE="${COORD_DIR}/endpoints.node${NODE_RANK}" |
| LOG_DIR="${REPO_ROOT}/logs/infinity_parser2_regeneration/4node_${coord_tag}" |
| mkdir -p "$COORD_DIR" "$LOG_DIR" |
| } |
|
|
| stop_vllm() { |
| local pid |
| local alive |
|
|
| |
| for pid in "${VLLM_PIDS[@]}"; do |
| kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true |
| done |
| for _ in {1..30}; do |
| alive=0 |
| for pid in "${VLLM_PIDS[@]}"; do |
| kill -0 "$pid" 2>/dev/null && alive=1 |
| done |
| (( alive == 0 )) && break |
| sleep 1 |
| done |
| for pid in "${VLLM_PIDS[@]}"; do |
| if kill -0 "$pid" 2>/dev/null; then |
| kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true |
| fi |
| wait "$pid" 2>/dev/null || true |
| done |
| VLLM_PIDS=() |
| } |
|
|
| cleanup() { |
| local status=$? |
| trap - EXIT INT TERM HUP |
|
|
| if (( COORD_ACTIVE == 1 )); then |
| if (( NODE_RANK == 0 )); then |
| touch "$STOP_FILE" |
| fi |
| stop_vllm |
| touch "$ACK_FILE" |
| if (( status != 0 )); then |
| printf 'node=%s status=%s\n' "$NODE_RANK" "$status" >"$FAILED_FILE" |
| fi |
| else |
| stop_vllm |
| fi |
| exit "$status" |
| } |
| trap cleanup EXIT |
| trap 'exit 130' INT |
| trap 'exit 143' TERM HUP |
|
|
| initialize_coordination() { |
| local rank |
|
|
| configure_runtime_paths |
| if (( NODE_RANK == 0 )); then |
| rm -f -- "$STOP_FILE" |
| for ((rank = 0; rank < NNODES; rank++)); do |
| rm -f -- \ |
| "${COORD_DIR}/stopped.node${rank}" \ |
| "${COORD_DIR}/failed.node${rank}" |
| done |
| fi |
| rm -f -- "$READY_FILE" |
| touch "$STARTED_FILE" |
| COORD_ACTIVE=1 |
| } |
|
|
| start_vllm() { |
| local index gpu port |
| local -a gpus=() |
| local -a args=( |
| "$VLLM_BIN" serve "$MODEL" |
| --served-model-name "$MODEL" |
| --tensor-parallel-size 1 |
| --host "$API_HOST" |
| --max-model-len "$MAX_MODEL_LEN" |
| --allowed-local-media-path "$MEDIA_ROOT" |
| --limit-mm-per-prompt "{\"image\":${MAX_IMAGES}}" |
| ) |
| local -a extra_args=() |
|
|
| if [[ -n "$MAX_NUM_SEQS" ]]; then |
| args+=(--max-num-seqs "$MAX_NUM_SEQS") |
| fi |
| if [[ -n "${PARSER2_TEACHER_GPU_IDS:-${CUDA_VISIBLE_DEVICES:-}}" ]]; then |
| IFS=',' read -r -a gpus <<<"${PARSER2_TEACHER_GPU_IDS:-${CUDA_VISIBLE_DEVICES}}" |
| else |
| for ((index = 0; index < LOCAL_DP_SIZE; index++)); do |
| gpus+=("$index") |
| done |
| fi |
| (( ${#gpus[@]} >= LOCAL_DP_SIZE )) || \ |
| die "PET_NPROC_PER_NODE=${LOCAL_DP_SIZE}, but only ${#gpus[@]} GPUs are configured" |
| if [[ -n "${PARSER2_VLLM_EXTRA_ARGS:-}" ]]; then |
| read -r -a extra_args <<<"$PARSER2_VLLM_EXTRA_ARGS" |
| args+=("${extra_args[@]}") |
| fi |
|
|
| echo "Starting node ${NODE_RANK}/${NNODES}: ${LOCAL_DP_SIZE} independent single-GPU servers on ports ${API_PORT}..$((API_PORT + LOCAL_DP_SIZE - 1))" |
| echo "Concurrency: ${CONCURRENCY_PER_GPU}/endpoint x ${GLOBAL_DP_SIZE} GPU endpoints = ${GLOBAL_CONCURRENCY} total; max-num-seqs=${MAX_NUM_SEQS:-vLLM default}/GPU" |
| |
| |
| |
| for ((index = 0; index < LOCAL_DP_SIZE; index++)); do |
| gpu="${gpus[$index]//[[:space:]]/}" |
| port=$((API_PORT + index)) |
| echo "GPU ${gpu}: ${LOG_DIR}/vllm_node${NODE_RANK}_gpu${index}.log" |
| setsid env \ |
| -u WORLD_SIZE -u RANK -u LOCAL_RANK -u LOCAL_WORLD_SIZE \ |
| -u MASTER_ADDR -u MASTER_PORT \ |
| CUDA_VISIBLE_DEVICES="$gpu" \ |
| VLLM_PLUGINS="" \ |
| HF_ENDPOINT="${HF_ENDPOINT:-https://hf-mirror.com}" \ |
| HF_HOME="${HF_HOME:-/inspire/sfs/project/inf-multimodal/public/wumengke/.cache/huggingface}" \ |
| PYTHONPATH="${REPO_ROOT}/hs_connectors/src${PYTHONPATH:+:${PYTHONPATH}}" \ |
| "${args[@]}" --port "$port" \ |
| >"${LOG_DIR}/vllm_node${NODE_RANK}_gpu${index}.log" 2>&1 & |
| VLLM_PIDS+=("$!") |
| done |
| } |
|
|
| check_local_servers() { |
| local index |
| for index in "${!VLLM_PIDS[@]}"; do |
| if ! kill -0 "${VLLM_PIDS[$index]}" 2>/dev/null; then |
| tail -n 120 "${LOG_DIR}/vllm_node${NODE_RANK}_gpu${index}.log" >&2 || true |
| die "node ${NODE_RANK} GPU ${index} vLLM process exited" |
| fi |
| done |
| } |
|
|
| new_failure_file() { |
| local failure |
| for failure in "${COORD_DIR}"/failed.node*; do |
| [[ -e "$failure" ]] || continue |
| if [[ "$failure" -nt "$STARTED_FILE" ]]; then |
| printf '%s\n' "$failure" |
| return 0 |
| fi |
| done |
| return 1 |
| } |
|
|
| wait_for_api() { |
| local deadline=$((SECONDS + START_TIMEOUT)) |
| local failure |
| local index ready |
|
|
| command -v curl >/dev/null || die "curl is required" |
| while true; do |
| check_local_servers |
| if failure="$(new_failure_file)"; then |
| cat "$failure" >&2 || true |
| die "a remote vLLM node exited during startup" |
| fi |
| ready=0 |
| for ((index = 0; index < LOCAL_DP_SIZE; index++)); do |
| if curl -fsS --max-time 5 "http://127.0.0.1:$((API_PORT + index))/health" >/dev/null 2>&1; then |
| ready=$((ready + 1)) |
| fi |
| done |
| (( ready == LOCAL_DP_SIZE )) && break |
| (( SECONDS < deadline )) || \ |
| die "vLLM startup timed out after ${START_TIMEOUT}s" |
| echo "Node ${NODE_RANK}: waiting for local APIs, ${ready}/${LOCAL_DP_SIZE} ready" |
| sleep 5 |
| done |
|
|
| for ((index = 0; index < LOCAL_DP_SIZE; index++)); do |
| printf 'http://%s:%s/v1/chat/completions\n' "$NODE_ADDRESS" "$((API_PORT + index))" |
| done >"${READY_FILE}.tmp" |
| mv -- "${READY_FILE}.tmp" "$READY_FILE" |
| echo "Node ${NODE_RANK}: ${LOCAL_DP_SIZE} APIs ready on ${NODE_ADDRESS}" |
| } |
|
|
| wait_for_cluster_apis() { |
| local deadline=$((SECONDS + START_TIMEOUT)) |
| local rank failure all_ready |
| local -a node_endpoints=() |
|
|
| while true; do |
| check_local_servers |
| if failure="$(new_failure_file)"; then |
| cat "$failure" >&2 || true |
| die "a remote vLLM node exited during startup" |
| fi |
| all_ready=1 |
| for ((rank = 0; rank < NNODES; rank++)); do |
| [[ -s "${COORD_DIR}/endpoints.node${rank}" ]] || all_ready=0 |
| done |
| (( all_ready == 1 )) && break |
| (( SECONDS < deadline )) || die "timed out waiting for all node APIs" |
| sleep 3 |
| done |
| ENDPOINTS=() |
| for ((rank = 0; rank < NNODES; rank++)); do |
| mapfile -t node_endpoints <"${COORD_DIR}/endpoints.node${rank}" |
| (( ${#node_endpoints[@]} == LOCAL_DP_SIZE )) || \ |
| die "node ${rank} published ${#node_endpoints[@]} endpoints, expected ${LOCAL_DP_SIZE}" |
| ENDPOINTS+=("${node_endpoints[@]}") |
| done |
| echo "Cluster ready: ${#ENDPOINTS[@]} GPU endpoints, ${CONCURRENCY_PER_GPU} concurrent requests each" |
| } |
|
|
| monitor_worker_node() { |
| while true; do |
| if [[ -e "$STOP_FILE" && "$STOP_FILE" -nt "$STARTED_FILE" ]]; then |
| stop_vllm |
| touch "$ACK_FILE" |
| COORD_ACTIVE=0 |
| echo "Node ${NODE_RANK} stopped after coordinator signal" |
| return 0 |
| fi |
| check_local_servers |
| sleep 3 |
| done |
| } |
|
|
| run_generation() { |
| local endpoint_list |
| endpoint_list="$(IFS=','; printf '%s' "${ENDPOINTS[*]}")" |
| PARSER2_PYTHON_BIN="$PYTHON_BIN" \ |
| PARSER2_SOURCE_JSONL="$SOURCE_JSONL" \ |
| PARSER2_MODEL="$MODEL" \ |
| PARSER2_MEDIA_ROOT="$MEDIA_ROOT" \ |
| PARSER2_DATA_ROOT="$DATA_ROOT" \ |
| PARSER2_REGEN_ROOT="$OUTPUT_ROOT" \ |
| PARSER2_FINAL_DIR="$FINAL_DIR" \ |
| PARSER2_PREPARED_ROOT="$PREPARED_ROOT" \ |
| PARSER2_ENDPOINTS="$endpoint_list" \ |
| PARSER2_MAX_TOKENS="$MAX_TOKENS" \ |
| PARSER2_CONCURRENCY_PER_ENDPOINT="$CONCURRENCY_PER_GPU" \ |
| PARSER2_SEQ_LENGTH="$SEQ_LENGTH" \ |
| bash "$RUN_ALL" generate full |
| } |
|
|
| wait_for_cluster_stop() { |
| local deadline=$((SECONDS + 180)) |
| local rank |
| local all_stopped |
|
|
| touch "$STOP_FILE" |
| stop_vllm |
| touch "$ACK_FILE" |
|
|
| while true; do |
| all_stopped=1 |
| for ((rank = 0; rank < NNODES; rank++)); do |
| if [[ ! -e "${COORD_DIR}/stopped.node${rank}" || \ |
| ! "${COORD_DIR}/stopped.node${rank}" -nt "$STOP_FILE" ]]; then |
| all_stopped=0 |
| break |
| fi |
| done |
| (( all_stopped == 1 )) && break |
| (( SECONDS < deadline )) || \ |
| die "timed out waiting for all nodes to stop; keeping ${COORD_DIR} for late workers" |
| sleep 3 |
| done |
|
|
| for ((rank = 0; rank < NNODES; rank++)); do |
| rm -f -- \ |
| "${COORD_DIR}/started.node${rank}" \ |
| "${COORD_DIR}/stopped.node${rank}" \ |
| "${COORD_DIR}/failed.node${rank}" \ |
| "${COORD_DIR}/endpoints.node${rank}" |
| done |
| rm -f -- "$STOP_FILE" |
| rmdir "$COORD_DIR" 2>/dev/null || true |
| COORD_ACTIVE=0 |
| } |
|
|
| run_prepare() { |
| PARSER2_PYTHON_BIN="$PYTHON_BIN" \ |
| PARSER2_SOURCE_JSONL="$SOURCE_JSONL" \ |
| PARSER2_MODEL="$MODEL" \ |
| PARSER2_MEDIA_ROOT="$MEDIA_ROOT" \ |
| PARSER2_DATA_ROOT="$DATA_ROOT" \ |
| PARSER2_REGEN_ROOT="$OUTPUT_ROOT" \ |
| PARSER2_FINAL_DIR="$FINAL_DIR" \ |
| PARSER2_PREPARED_ROOT="$PREPARED_ROOT" \ |
| PARSER2_PREPARE_ONLY=1 \ |
| PARSER2_PREPARE_MODE=fast \ |
| PARSER2_SEQ_LENGTH="$SEQ_LENGTH" \ |
| PARSER2_HOLD_GPUS="${PARSER2_HOLD_GPUS:-0}" \ |
| bash "$RUN_ALL" full |
| } |
|
|
| build_draft_vocab() { |
| local prepared_dir="${PREPARED_ROOT}/full" |
| local token_freq="${prepared_dir}/token_freq.pt" |
|
|
| require_file "$token_freq" |
| "$PYTHON_BIN" "$VOCAB_SCRIPT" \ |
| --token-freq-path "$token_freq" \ |
| --draft-vocab-size "$DRAFT_VOCAB_SIZE" \ |
| --target-model-path "$MODEL" \ |
| --output-path "$prepared_dir" |
|
|
| "$PYTHON_BIN" - \ |
| "${prepared_dir}/d2t.npy" \ |
| "${prepared_dir}/t2d.npy" \ |
| "$DRAFT_VOCAB_SIZE" <<'PY' |
| import sys |
| from pathlib import Path |
|
|
| import numpy as np |
|
|
| d2t_path, t2d_path = map(Path, sys.argv[1:3]) |
| draft_vocab_size = int(sys.argv[3]) |
| d2t = np.load(d2t_path, mmap_mode="r") |
| t2d = np.load(t2d_path, mmap_mode="r") |
| if d2t.shape != (draft_vocab_size,): |
| raise SystemExit(f"unexpected d2t shape: {d2t.shape}") |
| if t2d.ndim != 1: |
| raise SystemExit(f"unexpected t2d shape: {t2d.shape}") |
| print(f"Vocab mapping ready: d2t={d2t.shape}, t2d={t2d.shape}") |
| PY |
| } |
|
|
| show_status() { |
| PARSER2_PYTHON_BIN="$PYTHON_BIN" \ |
| PARSER2_MODEL="$MODEL" \ |
| PARSER2_DATA_ROOT="$DATA_ROOT" \ |
| PARSER2_REGEN_ROOT="$OUTPUT_ROOT" \ |
| bash "$RUN_ALL" status full |
| } |
|
|
| action="${1:-full}" |
| case "$action" in |
| full|generate) |
| validate_common_paths |
| require_executable "$VLLM_BIN" |
| command -v setsid >/dev/null || die "setsid is required" |
| validate_cluster_topology |
| initialize_coordination |
| resolve_node_address |
| start_vllm |
| wait_for_api |
| if (( NODE_RANK == 0 )); then |
| wait_for_cluster_apis |
| run_generation |
| wait_for_cluster_stop |
| if [[ "$action" == "full" ]]; then |
| run_prepare |
| build_draft_vocab |
| fi |
| else |
| monitor_worker_node |
| fi |
| ;; |
| prepare) |
| validate_common_paths |
| if [[ "${PET_NODE_RANK:-0}" == "0" ]]; then |
| run_prepare |
| build_draft_vocab |
| else |
| echo "Node ${PET_NODE_RANK}: prepare only runs on node 0" |
| fi |
| ;; |
| status) |
| validate_common_paths |
| if [[ "${PET_NODE_RANK:-0}" == "0" ]]; then |
| show_status |
| else |
| echo "Node ${PET_NODE_RANK}: status only runs on node 0" |
| fi |
| ;; |
| *) |
| echo "Usage: $(basename "$0") [full|generate|prepare|status]" >&2 |
| exit 2 |
| ;; |
| esac |
|
|
