Upload folder using huggingface_hub

performance_testing/1gpu.pbs

@@ -0,0 +1,94 @@
+#!/bin/bash
+#PBS -N 1gpu
+#PBS -l select=1:ncpus=20:mem=180gb:ngpus=1
+#PBS -q gpu_1d
+#PBS -o gpu_tests_log/1gpu_output.log
+#PBS -e gpu_tests_log/1gpu_error.log
+
+# === Load modules & activate conda ===
+module use /app/common/modules
+module load anaconda3-2024.10
+source activate gptoss-vllm
+
+# === Move to working directory ===
+cd /home/skiredj.abderrahman/khalil/performance_testing
+
+# === Confirm node ===
+echo "==== Job running on node: $(hostname -s) ===="
+
+# === Setup cleanup on job exit ===
+cleanup() {
+    echo "Cleaning up..."
+    kill 0
+}
+trap cleanup EXIT
+
+# === Start vmstat monitor (every 30s) ===
+monitor_vmstat() {
+    while true; do
+        echo "==== $(date) ====" >> gpu_tests_log/vmstat.txt
+        vmstat 1 2 | tail -1 >> gpu_tests_log/vmstat.txt
+        sleep 29
+    done
+}
+
+# === Start nvidia-smi monitor (every 30s) ===
+monitor_nvidia() {
+    while true; do
+
+        nvidia-smi \
+        --query-gpu=timestamp,utilization.gpu,memory.used,memory.total \
+        --format=csv,noheader,nounits >> gpu_tests_log/nvidia_smi.csv
+        sleep 30
+    done
+}
+
+# === Monitor vLLM RAM usage (swap-space %) ===
+monitor_vllm_metrics() {
+  OUT="gpu_tests_log/swap_space_usage.csv"
+  echo "ram_percent,rss_kb" > "$OUT"
+
+  TOTAL_RAM_KB=$((180 * 1024 * 1024))  # 180 GB in KB
+
+  while true; do
+    pid=$(pgrep -f 'vllm.*serve')
+    if [[ -n "$pid" ]]; then
+      rss=$(ps -o rss= -p "$pid" | awk '{print $1}')  # RSS in KB
+      ram_percent=$(awk -v r="$rss" -v t="$TOTAL_RAM_KB" 'BEGIN{printf "%.2f", (r/t)*100}')
+    else
+      rss=0
+      ram_percent=0
+    fi
+    echo "${ram_percent},${rss}" >> "$OUT"
+    sleep 30
+  done
+}
+
+
+# === Start vLLM server ===
+# inferior then 128k input + 8,192 output
+start_vllm() {
+
+    echo "==== Job running on node: $(hostname -s) ===="
+vllm serve /home/skiredj.abderrahman/fatnaoui/models/gptoss --host 0.0.0.0 --async-scheduling \
+  --port 9998  >> gpu_tests_log/vllm.log 2>&1
+  }
+
+# === Launch all 3 in background ===
+# === Launch server first ===
+start_vllm &                     # start vLLM in background
+
+# === Delay monitors by 90s (change to 60 or 120 as you like) ===
+DELAY_SEC=5   # set to 60 or 120 per your preference
+sleep "${DELAY_SEC}"
+
+# === Now start monitors ===
+monitor_vmstat &
+monitor_nvidia &
+monitor_vllm_metrics &
+
+# === Wait until everything exits ===
+wait
+
+# === Wait until job ends ===
+wait

performance_testing/2gpu.pbs

@@ -0,0 +1,102 @@
+#!/bin/bash
+#PBS -N 2gpu
+#PBS -l select=1:ncpus=20:mem=180gb:ngpus=1
+#PBS -q gpu_1d
+#PBS -o gpu_tests_log_second_node/1gpu_output.log
+#PBS -e gpu_tests_log_second_node/1gpu_error.log
+
+# === Load modules & activate conda ===
+module use /app/common/modules
+module load anaconda3-2024.10
+source activate gptoss-vllm
+
+# === Move to working directory ===
+cd /home/skiredj.abderrahman/khalil/performance_testing
+
+# === Confirm node ===
+echo "==== Job running on node: $(hostname -s) ===="
+
+# === Setup cleanup on job exit ===
+cleanup() {
+    echo "Cleaning up..."
+    kill 0
+}
+trap cleanup EXIT
+
+# === Start vmstat monitor (every 30s) ===
+monitor_vmstat() {
+    while true; do
+        echo "==== $(date) ====" >> gpu_tests_log_second_node/vmstat.txt
+        vmstat 1 2 | tail -1 >> gpu_tests_log_second_node/vmstat.txt
+        sleep 29
+    done
+}
+
+# === Start nvidia-smi monitor (every 30s) ===
+monitor_nvidia() {
+    while true; do
+
+        nvidia-smi \
+        --query-gpu=timestamp,utilization.gpu,memory.used,memory.total \
+        --format=csv,noheader,nounits >> gpu_tests_log_second_node/nvidia_smi.csv
+        sleep 30
+    done
+}
+
+# === Monitor vLLM RAM usage (swap-space %) ===
+monitor_vllm_metrics() {
+  OUT="gpu_tests_log_second_node/swap_space_usage.csv"
+  echo "ram_percent,rss_kb" > "$OUT"
+
+  TOTAL_RAM_KB=$((180 * 1024 * 1024))  # 180 GB in KB
+
+  while true; do
+    pid=$(pgrep -f 'vllm.*serve')
+    if [[ -n "$pid" ]]; then
+      rss=$(ps -o rss= -p "$pid" | awk '{print $1}')  # RSS in KB
+      ram_percent=$(awk -v r="$rss" -v t="$TOTAL_RAM_KB" 'BEGIN{printf "%.2f", (r/t)*100}')
+    else
+      rss=0
+      ram_percent=0
+    fi
+    echo "${ram_percent},${rss}" >> "$OUT"
+    sleep 30
+  done
+}
+
+
+# === Start vLLM server ===
+# inferior then 128k input + 8,192 output
+start_vllm() {
+
+    echo "==== Job running on node: $(hostname -s) ===="
+vllm serve /home/skiredj.abderrahman/fatnaoui/models/gptoss  \
+  --tensor-parallel-size 1 \
+  --port 9997  >> gpu_tests_log_second_node/vllm.log 2>&1
+  }
+
+# === Launch all 3 in background ===
+# === Launch server first ===
+start_vllm &                     # start vLLM in background
+VLLM_PID=$!
+
+# Wait until the metrics endpoint is up (vLLM ready)
+PORT=${PORT:-9998}
+until curl -s "http://localhost:${PORT}/metrics" >/dev/null 2>&1; do
+  sleep 2
+done
+
+# === Delay monitors by 90s (change to 60 or 120 as you like) ===
+DELAY_SEC=60   # set to 60 or 120 per your preference
+sleep "${DELAY_SEC}"
+
+# === Now start monitors ===
+monitor_vmstat &
+monitor_nvidia &
+monitor_vllm_metrics &
+
+# === Wait until everything exits ===
+wait
+
+# === Wait until job ends ===
+wait

performance_testing/gpt_gpu_monitoring.pbs

@@ -0,0 +1,77 @@
+#!/bin/bash
+#PBS -N khalil_gptoss_monitoring
+#PBS -l select=1:ncpus=20:mem=180gb:ngpus=1
+#PBS -q gpu_1d
+#PBS -o gpt_tests_log/gpt_output_%J.log
+#PBS -e gpt_tests_log/gpt_error_%J.log
+
+# === Load modules & activate conda ===
+module use /app/common/modules
+module load anaconda3-2024.10
+source activate gptoss-vllm
+
+# === Move to working directory ===
+cd /home/skiredj.abderrahman/khalil/performance_testing
+
+# === Confirm node ===
+echo "==== Job running on node: $(hostname -s) ===="
+
+# === Setup cleanup on job exit ===
+cleanup() {
+    echo "Cleaning up..."
+    kill 0
+}
+trap cleanup EXIT
+
+# === Start vmstat monitor (every 30s) ===
+monitor_vmstat() {
+    while true; do
+        echo "==== $(date) ====" >> gpt_tests_log/vmstat.txt
+        vmstat 1 2 | tail -1 >> gpt_tests_log/vmstat.txt
+        sleep 29
+    done
+}
+
+# === Start nvidia-smi monitor (every 30s) ===
+monitor_nvidia() {
+    while true; do
+
+        nvidia-smi \
+        --query-gpu=timestamp,utilization.gpu,memory.used,memory.total \
+        --format=csv,noheader,nounits >> gpt_tests_log/nvidia_smi.csv
+        sleep 30
+    done
+}
+
+# === Monitor vLLM RAM usage (swap-space %) ===
+monitor_vllm_metrics() {
+  OUT="gpt_tests_log/swap_space_usage.csv"
+  echo "ram_percent,rss_kb" > "$OUT"
+
+  TOTAL_RAM_KB=$((180 * 1024 * 1024))  # 180 GB in KB
+
+  while true; do
+    pid=$(pgrep -f 'vllm.*serve')
+    if [[ -n "$pid" ]]; then
+      rss=$(ps -o rss= -p "$pid" | awk '{print $1}')  # RSS in KB
+      ram_percent=$(awk -v r="$rss" -v t="$TOTAL_RAM_KB" 'BEGIN{printf "%.2f", (r/t)*100}')
+    else
+      rss=0
+      ram_percent=0
+    fi
+    echo "${ram_percent},${rss}" >> "$OUT"
+    sleep 30
+  done
+}
+
+
+# === Now start monitors ===
+monitor_vmstat &
+monitor_nvidia &
+monitor_vllm_metrics &
+
+# === Wait until everything exits ===
+wait
+
+# === Wait until job ends ===
+wait

performance_testing/gpt_tests_log/calcul_cpu_ram_gpu_%_per_30s.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Calculates CPU%, RAM% (host), GPU_Util% and VRAM_Usage% (GPU memory used/total)
+# Expects:
+#   vmstat.txt            -> your periodic vmstat dumps (same as before)
+#   nvidia_smi.csv        -> logged via nvidia-smi --query-gpu=...,noheader,nounits
+
+VMSTAT_FILE="vmstat.txt"
+GPU_CSV="nvidia_smi.csv"
+
+if [[ ! -f "$VMSTAT_FILE" ]]; then
+  echo "File $VMSTAT_FILE not found!"
+  exit 1
+fi
+if [[ ! -f "$GPU_CSV" ]]; then
+  echo "File $GPU_CSV not found! (expected CSV from nvidia-smi --query-gpu=...)"
+  exit 1
+fi
+
+# Total host memory in KB
+TOTAL_MEM=$(grep MemTotal /proc/meminfo | awk '{print $2}')
+
+echo "CPU_Usage%,Memory_Usage%,GPU_Util%,VRAM_Usage%"
+
+# --- Load & clean vmstat lines (skip headers/blank/separators) ---
+mapfile -t vmstat_lines < <(grep -vE 'procs|r[[:space:]]+b|^=|^$' "$VMSTAT_FILE")
+
+# --- Load GPU CSV rows (timestamp,util%,mem_used(MiB),mem_total(MiB)) ---
+mapfile -t gpu_rows < <(awk 'NF{gsub(/ /,""); print}' "$GPU_CSV")
+
+len=${#vmstat_lines[@]}
+for ((i = 0; i < len; i++)); do
+  line="${vmstat_lines[$i]}"
+
+  # Extract 'free' and 'id' fields from vmstat default format
+  # vmstat columns (Linux): r b swpd free buff cache si so bi bo in cs us sy id wa st
+  read -r _ _ _ free _ _ _ _ _ _ _ _ us sy id wa st <<< "$line"
+
+  cpu_usage=$((100 - id))
+  mem_used_kb=$((TOTAL_MEM - free))
+  mem_usage_pct=$(awk -v u="$mem_used_kb" -v t="$TOTAL_MEM" 'BEGIN{printf("%.2f", (u/t)*100)}')
+
+  # Match GPU row by index; if out of range, reuse last available
+  gpu_idx=$i
+  if (( gpu_idx >= ${#gpu_rows[@]} )); then
+    gpu_idx=$((${#gpu_rows[@]} - 1))
+  fi
+  gpu_row="${gpu_rows[$gpu_idx]}"
+
+  # gpu_row format after gsub spaces: YYYY/MM/DDHH:MM:SS,UTIL,MEM_USED,MEM_TOTAL
+  IFS=',' read -r _ts util mem_used_mib mem_total_mib <<< "$gpu_row"
+
+  # Compute VRAM usage %
+  if [[ -n "$mem_used_mib" && -n "$mem_total_mib" && "$mem_total_mib" -gt 0 ]]; then
+    vram_pct=$(awk -v u="$mem_used_mib" -v t="$mem_total_mib" 'BEGIN{printf("%.2f", (u/t)*100)}')
+  else
+    vram_pct="N/A"
+  fi
+
+  # If util is empty, set N/A
+  [[ -z "$util" ]] && util="N/A"
+
+  echo "${cpu_usage}%,$mem_usage_pct%,${util}%,$vram_pct%"
+done

performance_testing/gpt_tests_log/gpt_output_%J.log

@@ -0,0 +1,2 @@
+==== Job running on node: gpu03 ====
+Cleaning up...

performance_testing/gpt_tests_log/nvidia_smi.csv

@@ -0,0 +1,191 @@
+2026/02/10 18:16:59.900, 0, 0, 46068
+2026/02/10 18:17:30.443, 0, 0, 46068
+2026/02/10 18:18:00.985, 0, 0, 46068
+2026/02/10 18:18:31.536, 0, 0, 46068
+2026/02/10 18:19:02.085, 0, 0, 46068
+2026/02/10 18:19:32.628, 0, 0, 46068
+2026/02/10 18:20:03.162, 0, 0, 46068
+2026/02/10 18:20:33.705, 0, 0, 46068
+2026/02/10 18:21:03.818, 5, 15112, 46068
+2026/02/10 18:21:33.833, 0, 42614, 46068
+2026/02/10 18:22:03.848, 0, 42614, 46068
+2026/02/10 18:22:33.863, 0, 42614, 46068
+2026/02/10 18:23:03.878, 0, 42614, 46068
+2026/02/10 18:23:33.893, 0, 42614, 46068
+2026/02/10 18:24:03.908, 0, 42614, 46068
+2026/02/10 18:24:33.923, 0, 42614, 46068
+2026/02/10 18:25:03.938, 0, 42614, 46068
+2026/02/10 18:25:33.953, 0, 42614, 46068
+2026/02/10 18:26:03.967, 0, 42614, 46068
+2026/02/10 18:26:33.983, 0, 42614, 46068
+2026/02/10 18:27:03.998, 0, 42614, 46068
+2026/02/10 18:27:34.012, 0, 42614, 46068
+2026/02/10 18:28:04.027, 0, 42614, 46068
+2026/02/10 18:28:34.041, 0, 42614, 46068
+2026/02/10 18:29:04.056, 0, 42614, 46068
+2026/02/10 18:29:34.072, 0, 42614, 46068
+2026/02/10 18:30:04.087, 0, 42614, 46068
+2026/02/10 18:30:34.102, 0, 42614, 46068
+2026/02/10 18:31:04.117, 0, 42614, 46068
+2026/02/10 18:31:34.132, 0, 42614, 46068
+2026/02/10 18:32:04.147, 0, 42614, 46068
+2026/02/10 18:32:34.161, 0, 42614, 46068
+2026/02/10 18:33:04.176, 0, 42614, 46068
+2026/02/10 18:33:34.191, 0, 42614, 46068
+2026/02/10 18:34:04.205, 100, 42614, 46068
+2026/02/10 18:34:34.220, 100, 42614, 46068
+2026/02/10 18:35:04.235, 100, 42614, 46068
+2026/02/10 18:35:34.251, 100, 42614, 46068
+2026/02/10 18:36:04.266, 0, 42614, 46068
+2026/02/10 18:36:34.281, 0, 42614, 46068
+2026/02/10 18:37:04.296, 0, 42614, 46068
+2026/02/10 18:37:34.311, 0, 42614, 46068
+2026/02/10 18:38:04.327, 0, 42614, 46068
+2026/02/10 18:38:34.342, 0, 42614, 46068
+2026/02/10 18:39:04.357, 0, 42614, 46068
+2026/02/10 18:39:34.372, 0, 42614, 46068
+2026/02/10 18:40:04.387, 0, 42614, 46068
+2026/02/10 18:40:34.402, 0, 42614, 46068
+2026/02/10 18:41:04.523, 0, 42614, 46068
+2026/02/10 18:41:34.539, 0, 42614, 46068
+2026/02/10 18:42:04.554, 0, 42614, 46068
+2026/02/10 18:42:34.569, 0, 42614, 46068
+2026/02/10 18:43:04.584, 0, 42614, 46068
+2026/02/10 18:43:34.599, 0, 42614, 46068
+2026/02/10 18:44:04.614, 0, 42614, 46068
+2026/02/10 18:44:34.629, 0, 42614, 46068
+2026/02/10 18:45:04.644, 0, 42614, 46068
+2026/02/10 18:45:34.659, 0, 42614, 46068
+2026/02/10 18:46:04.673, 0, 42614, 46068
+2026/02/10 18:46:34.688, 0, 42614, 46068
+2026/02/10 18:47:04.703, 0, 42614, 46068
+2026/02/10 18:47:34.717, 0, 42614, 46068
+2026/02/10 18:48:04.732, 0, 42614, 46068
+2026/02/10 18:48:34.747, 0, 42614, 46068
+2026/02/10 18:49:04.762, 0, 42614, 46068
+2026/02/10 18:49:34.777, 0, 42614, 46068
+2026/02/10 18:50:04.792, 0, 42614, 46068
+2026/02/11 00:11:51.812, 0, 0, 46068
+2026/02/11 00:12:22.354, 0, 0, 46068
+2026/02/11 00:12:52.902, 0, 0, 46068
+2026/02/11 00:13:23.444, 0, 0, 46068
+2026/02/11 00:13:53.986, 0, 0, 46068
+2026/02/11 00:14:24.528, 0, 0, 46068
+2026/02/11 00:14:54.647, 0, 3, 46068
+2026/02/11 00:15:24.661, 0, 1000, 46068
+2026/02/11 00:15:54.676, 0, 1548, 46068
+2026/02/11 00:16:24.692, 0, 2644, 46068
+2026/02/11 00:16:54.706, 0, 2644, 46068
+2026/02/11 00:17:24.722, 0, 2644, 46068
+2026/02/11 00:17:55.218, 0, 0, 46068
+2026/02/11 00:18:25.335, 68, 3454, 46068
+2026/02/11 00:18:55.356, 0, 3460, 46068
+2026/02/11 00:19:25.371, 0, 3460, 46068
+2026/02/11 00:19:55.387, 0, 3460, 46068
+2026/02/11 00:20:25.401, 0, 4536, 46068
+2026/02/11 00:20:55.417, 0, 4536, 46068
+2026/02/11 00:21:25.432, 0, 4536, 46068
+2026/02/11 00:21:55.448, 0, 4536, 46068
+2026/02/11 00:22:25.464, 0, 4536, 46068
+2026/02/11 00:22:55.479, 0, 4536, 46068
+2026/02/11 00:23:25.495, 0, 4536, 46068
+2026/02/11 00:23:55.510, 0, 4536, 46068
+2026/02/11 00:24:25.526, 0, 4536, 46068
+2026/02/11 00:24:55.542, 0, 4536, 46068
+2026/02/11 00:25:25.557, 0, 4536, 46068
+2026/02/11 00:25:55.573, 0, 4536, 46068
+2026/02/11 00:26:25.588, 0, 4536, 46068
+2026/02/11 00:26:55.604, 0, 4536, 46068
+2026/02/11 00:27:25.619, 0, 4536, 46068
+2026/02/11 00:27:55.635, 0, 4536, 46068
+2026/02/11 00:28:25.651, 0, 4536, 46068
+2026/02/11 00:28:55.666, 0, 4536, 46068
+2026/02/11 00:29:25.681, 0, 4536, 46068
+2026/02/11 00:29:55.697, 0, 4536, 46068
+2026/02/11 00:30:25.712, 0, 4536, 46068
+2026/02/11 00:30:55.728, 0, 4536, 46068
+2026/02/11 00:31:25.743, 0, 4536, 46068
+2026/02/11 00:31:55.759, 0, 4536, 46068
+2026/02/11 00:32:25.774, 0, 4536, 46068
+2026/02/11 00:32:55.790, 0, 4536, 46068
+2026/02/11 00:33:25.806, 0, 4536, 46068
+2026/02/11 00:33:55.821, 0, 4536, 46068
+2026/02/11 00:34:25.837, 0, 4536, 46068
+2026/02/11 00:34:55.852, 0, 4536, 46068
+2026/02/11 00:35:25.868, 0, 4536, 46068
+2026/02/11 00:35:55.884, 0, 4536, 46068
+2026/02/11 00:36:25.900, 0, 4536, 46068
+2026/02/11 00:36:55.915, 0, 4536, 46068
+2026/02/11 00:37:25.931, 0, 4536, 46068
+2026/02/11 00:37:55.946, 0, 4536, 46068
+2026/02/11 00:38:25.961, 0, 4536, 46068
+2026/02/11 00:38:55.977, 0, 4536, 46068
+2026/02/11 00:39:25.992, 0, 4536, 46068
+2026/02/11 00:39:56.008, 0, 4536, 46068
+2026/02/11 00:40:26.023, 0, 4536, 46068
+2026/02/11 00:40:56.038, 0, 4536, 46068
+2026/02/11 00:41:26.053, 0, 4536, 46068
+2026/02/11 00:41:56.069, 0, 4536, 46068
+2026/02/11 00:42:26.084, 0, 4536, 46068
+2026/02/11 00:42:56.100, 0, 4536, 46068
+2026/02/11 00:43:26.115, 0, 4536, 46068
+2026/02/11 00:43:56.131, 0, 4536, 46068
+2026/02/11 00:44:26.146, 0, 4536, 46068
+2026/02/11 00:44:56.162, 0, 4536, 46068
+2026/02/11 00:45:26.177, 0, 4536, 46068
+2026/02/11 00:45:56.193, 0, 4536, 46068
+2026/02/11 00:46:26.208, 0, 4536, 46068
+2026/02/11 00:46:56.224, 0, 4536, 46068
+2026/02/11 00:47:26.240, 0, 4536, 46068
+2026/02/11 00:47:56.255, 0, 4536, 46068
+2026/02/11 00:48:26.270, 0, 4536, 46068
+2026/02/11 00:48:56.286, 0, 4536, 46068
+2026/02/11 00:49:26.302, 0, 4536, 46068
+2026/02/11 00:49:56.317, 0, 4536, 46068
+2026/02/11 00:50:26.333, 0, 4536, 46068
+2026/02/11 00:50:56.349, 0, 4536, 46068
+2026/02/11 00:51:26.364, 0, 4536, 46068
+2026/02/11 00:51:56.379, 0, 4536, 46068
+2026/02/11 00:52:26.395, 0, 4536, 46068
+2026/02/11 00:52:56.411, 0, 4536, 46068
+2026/02/11 00:53:26.427, 0, 4536, 46068
+2026/02/11 00:53:56.443, 0, 4536, 46068
+2026/02/11 00:54:26.458, 0, 4536, 46068
+2026/02/11 00:54:56.474, 0, 4536, 46068
+2026/02/11 00:55:26.489, 0, 4536, 46068
+2026/02/11 00:55:56.505, 0, 4536, 46068
+2026/02/11 00:56:26.521, 0, 4536, 46068
+2026/02/11 00:56:56.536, 0, 4536, 46068
+2026/02/11 00:57:26.552, 0, 4536, 46068
+2026/02/11 00:57:56.567, 0, 4536, 46068
+2026/02/11 00:58:26.583, 0, 4536, 46068
+2026/02/11 00:58:56.598, 0, 4536, 46068
+2026/02/11 00:59:26.614, 0, 4536, 46068
+2026/02/11 00:59:56.630, 0, 4536, 46068
+2026/02/11 01:00:26.645, 0, 4536, 46068
+2026/02/11 01:00:56.660, 0, 4536, 46068
+2026/02/11 01:01:26.676, 0, 4536, 46068
+2026/02/11 01:01:56.691, 0, 4536, 46068
+2026/02/11 01:02:26.711, 0, 4536, 46068
+2026/02/11 01:02:56.727, 0, 4536, 46068
+2026/02/11 01:03:26.742, 0, 4536, 46068
+2026/02/11 01:03:56.758, 0, 4536, 46068
+2026/02/11 01:04:26.774, 0, 4536, 46068
+2026/02/11 01:04:56.789, 0, 4536, 46068
+2026/02/11 01:05:26.805, 0, 4536, 46068
+2026/02/11 01:05:56.821, 0, 4536, 46068
+2026/02/11 01:06:26.836, 0, 4536, 46068
+2026/02/11 01:06:56.852, 0, 4536, 46068
+2026/02/11 01:07:26.867, 0, 4536, 46068
+2026/02/11 01:07:56.883, 0, 4536, 46068
+2026/02/11 01:08:26.899, 0, 4536, 46068
+2026/02/11 01:08:56.914, 0, 4536, 46068
+2026/02/11 01:09:26.930, 0, 4536, 46068
+2026/02/11 01:09:56.947, 0, 4536, 46068
+2026/02/11 01:10:26.964, 0, 4536, 46068
+2026/02/11 01:10:56.981, 0, 4536, 46068
+2026/02/11 01:11:26.998, 0, 4536, 46068
+2026/02/11 01:11:57.015, 0, 4536, 46068
+2026/02/11 01:12:27.033, 0, 4536, 46068
+2026/02/11 01:12:57.050, 0, 4536, 46068
+2026/02/11 01:13:27.490, 0, 0, 46068

performance_testing/gpt_tests_log/swap_space_usage.csv

@@ -0,0 +1,125 @@
+ram_percent,rss_kb
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0
+0,0

performance_testing/gpt_tests_log/vmstat.txt

@@ -0,0 +1,382 @@
+==== Tue Feb 10 06:16:59 PM +01 2026 ====
+ 0  0      0 189684480   3836 1019972    0    0     0     0 2357  475  0  3 97  0  0
+==== Tue Feb 10 06:17:29 PM +01 2026 ====
+ 2  0      0 189655072   3836 1020464    0    0     0     4 1504  341  0  2 98  0  0
+==== Tue Feb 10 06:17:59 PM +01 2026 ====
+ 0  0      0 189689280   3836 1020952    0    0     0     0  225  139  0  0 100  0  0
+==== Tue Feb 10 06:18:29 PM +01 2026 ====
+ 0  0      0 189700320   3836 1021376    0    0     0     0  252  144  0  0 100  0  0
+==== Tue Feb 10 06:18:59 PM +01 2026 ====
+ 0  0      0 189714208   3836 1021856    0    0     0     0  199  157  0  0 100  0  0
+==== Tue Feb 10 06:19:29 PM +01 2026 ====
+ 0  0      0 189712368   3836 1022408    0    0     0     0  232  144  0  0 100  0  0
+==== Tue Feb 10 06:19:59 PM +01 2026 ====
+ 0  0      0 189711120   3836 1022776    0    0     0     0  307  187  0  0 100  0  0
+==== Tue Feb 10 06:20:29 PM +01 2026 ====
+ 0  0      0 189692416   3836 1023580    0    0     0     0  218  148  0  0 100  0  0
+==== Tue Feb 10 06:20:59 PM +01 2026 ====
+ 1  0      0 186904528   3836 1896216    0    0   216     0 25078 25757  4  2 93  2  0
+==== Tue Feb 10 06:21:29 PM +01 2026 ====
+ 0  0      0 171323984   3836 15920804    0    0     0     0 1246 1936  0  0 100  0  0
+==== Tue Feb 10 06:21:59 PM +01 2026 ====
+ 0  0      0 171327056   3836 15921256    0    0     0     0 1259 1905  0  0 100  0  0
+==== Tue Feb 10 06:22:29 PM +01 2026 ====
+ 0  0      0 171322160   3836 15923192    0    0     0     0 1229 1899  0  0 100  0  0
+==== Tue Feb 10 06:22:59 PM +01 2026 ====
+ 0  0      0 171310576   3836 15923580    0    0     0     0 1214 1906  0  0 100  0  0
+==== Tue Feb 10 06:23:29 PM +01 2026 ====
+ 0  0      0 171297472   3836 15924140    0    0     0     0 1245 1913  0  0 100  0  0
+==== Tue Feb 10 06:23:59 PM +01 2026 ====
+ 0  0      0 171306752   3836 15924508    0    0     0     0 1187 1864  0  0 100  0  0
+==== Tue Feb 10 06:24:29 PM +01 2026 ====
+ 0  0      0 171304000   3836 15925068    0    0     0     0 1219 1870  0  0 100  0  0
+==== Tue Feb 10 06:24:59 PM +01 2026 ====
+ 0  0      0 171309776   3836 15925440    0    0     0     0 1215 1873  0  0 100  0  0
+==== Tue Feb 10 06:25:29 PM +01 2026 ====
+ 0  0      0 171317184   3836 15926008    0    0     0     0 1297 1874  0  0 100  0  0
+==== Tue Feb 10 06:25:59 PM +01 2026 ====
+ 0  0      0 171303744   3836 15926408    0    0     0     0 1329 1898  0  0 100  0  0
+==== Tue Feb 10 06:26:29 PM +01 2026 ====
+ 0  0      0 171299168   3836 15926960    0    0     0     0 1344 1858  0  0 100  0  0
+==== Tue Feb 10 06:26:59 PM +01 2026 ====
+ 0  0      0 171310608   3836 15927332    0    0     0     0 1244 1874  0  0 100  0  0
+==== Tue Feb 10 06:27:29 PM +01 2026 ====
+ 0  0      0 171292704   3836 15927896    0    0     0     0 1175 1859  0  0 100  0  0
+==== Tue Feb 10 06:27:59 PM +01 2026 ====
+ 0  0      0 171295168   3836 15928264    0    0     0     0 1207 1874  0  0 100  0  0
+==== Tue Feb 10 06:28:29 PM +01 2026 ====
+ 0  0      0 171292336   3836 15928824    0    0     0     0 1197 1838  0  0 100  0  0
+==== Tue Feb 10 06:28:59 PM +01 2026 ====
+ 0  0      0 171284736   3836 15929220    0    0     0     0 1227 1903  0  0 100  0  0
+==== Tue Feb 10 06:29:29 PM +01 2026 ====
+ 0  0      0 171276848   3836 15929776    0    0     0     0 1167 1838  0  0 100  0  0
+==== Tue Feb 10 06:29:59 PM +01 2026 ====
+ 0  0      0 171287728   3836 15930144    0    0     0     0 1212 1883  0  0 100  0  0
+==== Tue Feb 10 06:30:29 PM +01 2026 ====
+ 0  0      0 171284096   3836 15930788    0    0     0     0 1192 1841  0  0 100  0  0
+==== Tue Feb 10 06:30:59 PM +01 2026 ====
+ 0  0      0 171291504   3836 15931036    0    0     0     0 1359 1975  0  0 100  0  0
+==== Tue Feb 10 06:31:29 PM +01 2026 ====
+ 0  0      0 171289696   3836 15931712    0    0     0     0 1176 1849  0  0 100  0  0
+==== Tue Feb 10 06:31:59 PM +01 2026 ====
+ 0  0      0 171301440   3836 15932080    0    0     0     0 1201 1879  0  0 100  0  0
+==== Tue Feb 10 06:32:29 PM +01 2026 ====
+ 0  0      0 171288240   3836 15932636    0    0     0     0 1150 1842  0  0 100  0  0
+==== Tue Feb 10 06:32:59 PM +01 2026 ====
+ 0  0      0 171289792   3836 15933008    0    0     0     0 1215 1875  0  0 100  0  0
+==== Tue Feb 10 06:33:29 PM +01 2026 ====
+ 0  0      0 171272224   3836 15933740    0    0     0     0 1374 1860  0  0 100  0  0
+==== Tue Feb 10 06:33:59 PM +01 2026 ====
+ 0  0      0 171267328   3836 15934104    0    0     0     0 1196 1890  0  0 100  0  0
+==== Tue Feb 10 06:34:29 PM +01 2026 ====
+ 0  0      0 171256896   3836 15934652    0    0     0     0 1174 1864  0  0 100  0  0
+==== Tue Feb 10 06:34:59 PM +01 2026 ====
+ 0  0      0 171225072   3836 15935024    0    0     0     0 1232 1884  0  0 100  0  0
+==== Tue Feb 10 06:35:29 PM +01 2026 ====
+ 0  0      0 171216192   3836 15935584    0    0     0     0 1185 1867  0  0 100  0  0
+==== Tue Feb 10 06:35:59 PM +01 2026 ====
+ 0  0      0 171213696   3836 15935952    0    0     0     0 1262 1883  0  0 100  0  0
+==== Tue Feb 10 06:36:29 PM +01 2026 ====
+ 0  0      0 171192384   3836 15936504    0    0     0     0 1263 1877  0  0 100  0  0
+==== Tue Feb 10 06:36:59 PM +01 2026 ====
+ 0  0      0 171197984   3836 15936876    0    0     0     0 1186 1880  0  0 100  0  0
+==== Tue Feb 10 06:37:29 PM +01 2026 ====
+ 0  0      0 171182272   3836 15937424    0    0     0     0 1237 1866  0  0 100  0  0
+==== Tue Feb 10 06:37:59 PM +01 2026 ====
+ 0  0      0 171170016   3836 15937792    0    0     0     0 1270 1890  0  0 100  0  0
+==== Tue Feb 10 06:38:29 PM +01 2026 ====
+ 0  0      0 171163408   3836 15938344    0    0     0     0 1177 1863  0  0 100  0  0
+==== Tue Feb 10 06:38:59 PM +01 2026 ====
+ 0  0      0 171147872   3836 15938712    0    0     0     0 1233 1898  0  0 100  0  0
+==== Tue Feb 10 06:39:29 PM +01 2026 ====
+ 0  0      0 171125856   3836 15939268    0    0     0     0 1178 1857  0  0 100  0  0
+==== Tue Feb 10 06:39:59 PM +01 2026 ====
+ 0  0      0 171106256   3836 15939636    0    0     0     0 1220 1887  0  0 100  0  0
+==== Tue Feb 10 06:40:29 PM +01 2026 ====
+ 0  0      0 171093248   3836 15940268    0    0     0     0 1345 1860  0  0 100  0  0
+==== Tue Feb 10 06:40:59 PM +01 2026 ====
+ 0  0      0 171091264   3836 15940640    0    0     0     0 1289 1956  0  0 100  0  0
+==== Tue Feb 10 06:41:29 PM +01 2026 ====
+ 0  0      0 171098816   3836 15941160    0    0     0     0 1200 1858  0  0 100  0  0
+==== Tue Feb 10 06:41:59 PM +01 2026 ====
+ 0  0      0 171086656   3836 15941508    0    0     0     0 1223 1865  0  0 100  0  0
+==== Tue Feb 10 06:42:29 PM +01 2026 ====
+ 0  0      0 171080704   3836 15942060    0    0     0     0 1176 1843  0  0 100  0  0
+==== Tue Feb 10 06:42:59 PM +01 2026 ====
+ 0  0      0 171078560   3836 15942432    0    0     0     0 1188 1891  0  0 100  0  0
+==== Tue Feb 10 06:43:30 PM +01 2026 ====
+ 0  0      0 171085120   3836 15942984    0    0     0     0 1228 1908  0  0 100  0  0
+==== Tue Feb 10 06:44:00 PM +01 2026 ====
+ 0  0      0 171074464   3836 15943352    0    0     0     0 1188 1896  0  0 100  0  0
+==== Tue Feb 10 06:44:30 PM +01 2026 ====
+ 0  0      0 171080416   3836 15943904    0    0     0     0 1231 1872  0  0 100  0  0
+==== Tue Feb 10 06:45:00 PM +01 2026 ====
+ 0  0      0 171083376   3836 15944272    0    0     0     0 1195 1906  0  0 100  0  0
+==== Tue Feb 10 06:45:30 PM +01 2026 ====
+ 0  0      0 171090448   3836 15944836    0    0     0     0 1201 1873  0  0 100  0  0
+==== Tue Feb 10 06:46:00 PM +01 2026 ====
+ 0  0      0 171096896   3836 15945204    0    0     0     0 1278 1918  0  0 100  0  0
+==== Tue Feb 10 06:46:30 PM +01 2026 ====
+ 0  0      0 171098720   3836 15945760    0    0     0     0 1172 1869  0  0 100  0  0
+==== Tue Feb 10 06:47:00 PM +01 2026 ====
+ 0  0      0 171096480   3836 15946128    0    0     0     0 1250 1898  0  0 100  0  0
+==== Tue Feb 10 06:47:30 PM +01 2026 ====
+ 0  0      0 171090896   3836 15946680    0    0     0     0 1218 1879  0  0 100  0  0
+==== Tue Feb 10 06:48:00 PM +01 2026 ====
+ 0  0      0 171079728   3836 15947048    0    0     0     0 1301 1921  0  0 100  0  0
+==== Tue Feb 10 06:48:30 PM +01 2026 ====
+ 0  0      0 171081360   3836 15947600    0    0     0     0 1176 1866  0  0 100  0  0
+==== Tue Feb 10 06:49:00 PM +01 2026 ====
+ 0  0      0 171081760   3836 15947972    0    0     0     0 1198 1903  0  0 100  0  0
+==== Tue Feb 10 06:49:30 PM +01 2026 ====
+ 0  0      0 171079904   3836 15948528    0    0     0     0 1191 1867  0  0 100  0  0
+==== Tue Feb 10 06:50:00 PM +01 2026 ====
+ 0  0      0 171077680   3836 15948896    0    0     0     0 1258 1886  0  0 100  0  0
+==== Wed Feb 11 12:11:51 AM +01 2026 ====
+ 0  0      0 173770048   3836 16932684    0    0     0     0 2265  457  0  3 97  0  0
+==== Wed Feb 11 12:12:21 AM +01 2026 ====
+ 2  0      0 173744288   3836 16933236    0    0     0     0 1743  356  0  3 97  0  0
+==== Wed Feb 11 12:12:51 AM +01 2026 ====
+ 0  0      0 173779824   3836 16933608    0    0     0     0  283  144  0  0 100  0  0
+==== Wed Feb 11 12:13:21 AM +01 2026 ====
+ 1  0      0 173769152   3836 16934236    0    0     0     0  424  421  0  0 100  0  0
+==== Wed Feb 11 12:13:51 AM +01 2026 ====
+ 1  0      0 173770016   3836 16934604    0    0     0     0  275  200  0  0 100  0  0
+==== Wed Feb 11 12:14:21 AM +01 2026 ====
+ 1  0      0 173683872   3836 16935204    0    0     0     0  306  205  0  0 100  0  0
+==== Wed Feb 11 12:14:51 AM +01 2026 ====
+ 0  0      0 173172752   3836 16934948    0    0     0     4  959  836  0  0 100  0  0
+==== Wed Feb 11 12:15:21 AM +01 2026 ====
+ 0  0      0 172390432   3836 17501732    0    0     0     0 1094  980  0  0 100  0  0
+==== Wed Feb 11 12:15:51 AM +01 2026 ====
+ 0  0      0 172314432   3836 17513492    0    0     0     4  686  596  0  0 100  0  0
+==== Wed Feb 11 12:16:21 AM +01 2026 ====
+ 0  0      0 172272560   3836 17514172    0    0     0     0 7452 12709  2  0 98  0  0
+==== Wed Feb 11 12:16:51 AM +01 2026 ====
+ 0  0      0 172241856   3836 17515220    0    0     0     0  570  557  0  0 100  0  0
+==== Wed Feb 11 12:17:21 AM +01 2026 ====
+ 0  0      0 172231664   3836 17515900    0    0     0     0  516  531  0  0 100  0  0
+==== Wed Feb 11 12:17:51 AM +01 2026 ====
+ 0  0      0 173095776   3836 17507332    0    0     0     0  584  535  0  0 100  0  0
+==== Wed Feb 11 12:18:21 AM +01 2026 ====
+ 0  0      0 172598704   3836 17507768    0    0     0     0  487  508  0  0 100  0  0
+==== Wed Feb 11 12:18:51 AM +01 2026 ====
+ 0  0      0 171252784   3836 18140172    0    0     0     0  560  550  0  0 100  0  0
+==== Wed Feb 11 12:19:21 AM +01 2026 ====
+ 0  0      0 171187872   3836 18140728    0    0     0     0  553  539  0  0 100  0  0
+==== Wed Feb 11 12:19:51 AM +01 2026 ====
+ 0  0      0 171174304   3836 18141096    0    0     0     0  641  600  0  0 100  0  0
+==== Wed Feb 11 12:20:21 AM +01 2026 ====
+ 0  0      0 170336944   3836 18163008    0    0     0     0  537  563  0  0 100  0  0
+==== Wed Feb 11 12:20:51 AM +01 2026 ====
+ 0  0      0 170338624   3836 18163376    0    0     0     0 1700 1542  0  0 100  0  0
+==== Wed Feb 11 12:21:21 AM +01 2026 ====
+ 0  0      0 170332400   3836 18163216    0    0     0     0 1045 1017  0  0 100  0  0
+==== Wed Feb 11 12:21:51 AM +01 2026 ====
+ 0  0      0 170343680   3836 18163572    0    0     0     4 1648 1609  0  0 100  0  0
+==== Wed Feb 11 12:22:21 AM +01 2026 ====
+ 0  0      0 170344704   3836 18164128    0    0     0     0 1043 1019  0  0 100  0  0
+==== Wed Feb 11 12:22:51 AM +01 2026 ====
+ 0  0      0 170321472   3836 18164612    0    0     0     0  770  635  0  0 100  0  0
+==== Wed Feb 11 12:23:21 AM +01 2026 ====
+ 0  0      0 170289872   3836 18165048    0    0     0     0  657  591  0  0 100  0  0
+==== Wed Feb 11 12:23:51 AM +01 2026 ====
+ 0  0      0 170290416   3836 18165416    0    0     0     0  615  627  0  0 100  0  0
+==== Wed Feb 11 12:24:21 AM +01 2026 ====
+ 0  0      0 170267648   3836 18165972    0    0     0     0  534  569  0  0 100  0  0
+==== Wed Feb 11 12:24:51 AM +01 2026 ====
+ 0  0      0 170267968   3836 18166340    0    0     0     0  807  825  0  0 100  0  0
+==== Wed Feb 11 12:25:21 AM +01 2026 ====
+ 0  0      0 170264544   3836 18166844    0    0     0     0  572  539  0  0 100  0  0
+==== Wed Feb 11 12:25:51 AM +01 2026 ====
+ 0  0      0 170232800   3836 18167244    0    0     0    12  593  612  0  0 100  0  0
+==== Wed Feb 11 12:26:21 AM +01 2026 ====
+ 0  0      0 170164544   3836 18167924    0    0     0     0  523  422  0  0 100  0  0
+==== Wed Feb 11 12:26:51 AM +01 2026 ====
+ 0  0      0 170139968   3836 18168292    0    0     0     0  545  430  0  0 100  0  0
+==== Wed Feb 11 12:27:21 AM +01 2026 ====
+ 0  0      0 170125312   3836 18168848    0    0     0     0  508  431  0  0 100  0  0
+==== Wed Feb 11 12:27:51 AM +01 2026 ====
+ 0  0      0 170107472   3836 18169092    0    0     0     0  481  383  0  0 100  0  0
+==== Wed Feb 11 12:28:21 AM +01 2026 ====
+ 0  0      0 170083120   3836 18169768    0    0     0     0  516  378  0  0 100  0  0
+==== Wed Feb 11 12:28:51 AM +01 2026 ====
+ 0  0      0 170058960   3836 18170016    0    0     0     0  544  393  0  0 100  0  0
+==== Wed Feb 11 12:29:21 AM +01 2026 ====
+ 0  0      0 170037632   3836 18170716    0    0     0     0  516  358  0  0 100  0  0
+==== Wed Feb 11 12:29:51 AM +01 2026 ====
+ 0  0      0 170023568   3836 18170956    0    0     0     0  487  322  0  0 100  0  0
+==== Wed Feb 11 12:30:21 AM +01 2026 ====
+ 0  0      0 170015648   3836 18171584    0    0     0     0  488  300  0  0 100  0  0
+==== Wed Feb 11 12:30:51 AM +01 2026 ====
+ 0  0      0 170000800   3836 18171952    0    0     0     0  397  322  0  0 100  0  0
+==== Wed Feb 11 12:31:21 AM +01 2026 ====
+ 0  0      0 169994720   3836 18172508    0    0     0     0  382  282  0  0 100  0  0
+==== Wed Feb 11 12:31:51 AM +01 2026 ====
+ 0  0      0 169985152   3836 18172896    0    0     0     0  400  330  0  0 100  0  0
+==== Wed Feb 11 12:32:21 AM +01 2026 ====
+ 0  0      0 169982464   3836 18173656    0    0     0     0  407  292  0  0 100  0  0
+==== Wed Feb 11 12:32:51 AM +01 2026 ====
+ 0  0      0 169974208   3836 18173900    0    0     0     0  382  307  0  0 100  0  0
+==== Wed Feb 11 12:33:21 AM +01 2026 ====
+ 0  0      0 169984032   3836 18174452    0    0     0     0  408  312  0  0 100  0  0
+==== Wed Feb 11 12:33:51 AM +01 2026 ====
+ 0  0      0 169982928   3836 18174944    0    0     0     0  387  314  0  0 100  0  0
+==== Wed Feb 11 12:34:21 AM +01 2026 ====
+ 0  0      0 169993216   3836 18175376    0    0     0     0  463  317  0  0 100  0  0
+==== Wed Feb 11 12:34:51 AM +01 2026 ====
+ 0  0      0 169992160   3836 18175744    0    0     0     0  389  315  0  0 100  0  0
+==== Wed Feb 11 12:35:21 AM +01 2026 ====
+ 0  0      0 169988800   3836 18176432    0    0     0     0  398  306  0  0 100  0  0
+==== Wed Feb 11 12:35:51 AM +01 2026 ====
+ 0  0      0 169987760   3836 18176676    0    0     0     0  371  305  0  0 100  0  0
+==== Wed Feb 11 12:36:21 AM +01 2026 ====
+ 0  0      0 169985120   3836 18177348    0    0     0     0  404  320  0  0 100  0  0
+==== Wed Feb 11 12:36:51 AM +01 2026 ====
+ 0  0      0 169984768   3836 18177596    0    0     0     0  385  307  0  0 100  0  0
+==== Wed Feb 11 12:37:21 AM +01 2026 ====
+ 0  0      0 169981952   3836 18178276    0    0     0     0  440  349  0  0 100  0  0
+==== Wed Feb 11 12:37:51 AM +01 2026 ====
+ 0  0      0 169981024   3836 18178520    0    0     0     0  395  317  0  0 100  0  0
+==== Wed Feb 11 12:38:21 AM +01 2026 ====
+ 0  0      0 169978352   3836 18179072    0    0     0     0  404  301  0  0 100  0  0
+==== Wed Feb 11 12:38:51 AM +01 2026 ====
+ 0  0      0 169978048   3836 18179440    0    0     0     0  398  313  0  0 100  0  0
+==== Wed Feb 11 12:39:21 AM +01 2026 ====
+ 0  0      0 169976288   3836 18179992    0    0     0     0  400  301  0  0 100  0  0
+==== Wed Feb 11 12:39:51 AM +01 2026 ====
+ 0  0      0 169973824   3836 18180480    0    0     0     0  397  316  0  0 100  0  0
+==== Wed Feb 11 12:40:21 AM +01 2026 ====
+ 0  0      0 169988960   3836 18181112    0    0     0     0  429  328  0  0 100  0  0
+==== Wed Feb 11 12:40:51 AM +01 2026 ====
+ 0  0      0 169991312   3836 18181480    0    0     0     0  390  328  0  0 100  0  0
+==== Wed Feb 11 12:41:21 AM +01 2026 ====
+ 0  0      0 169989200   3836 18182036    0    0     0     0  439  331  0  0 100  0  0
+==== Wed Feb 11 12:41:51 AM +01 2026 ====
+ 0  0      0 169988096   3836 18182408    0    0     0     0  387  321  0  0 100  0  0
+==== Wed Feb 11 12:42:21 AM +01 2026 ====
+ 0  0      0 169997008   3836 18182956    0    0     0     0  401  323  0  0 100  0  0
+==== Wed Feb 11 12:42:51 AM +01 2026 ====
+ 0  0      0 169995424   3836 18183324    0    0     0     0  402  332  0  0 100  0  0
+==== Wed Feb 11 12:43:21 AM +01 2026 ====
+ 0  0      0 169993472   3836 18183896    0    0     0     0  411  320  0  0 100  0  0
+==== Wed Feb 11 12:43:51 AM +01 2026 ====
+ 0  0      0 169992240   3836 18184264    0    0     0     0  398  326  0  0 100  0  0
+==== Wed Feb 11 12:44:21 AM +01 2026 ====
+ 0  0      0 169980320   3836 18184820    0    0     0     0  425  352  0  0 100  0  0
+==== Wed Feb 11 12:44:51 AM +01 2026 ====
+ 0  0      0 169978560   3836 18185192    0    0     0     0  448  346  0  0 100  0  0
+==== Wed Feb 11 12:45:21 AM +01 2026 ====
+ 0  0      0 169979168   3836 18185752    0    0     0     0  380  315  0  0 100  0  0
+==== Wed Feb 11 12:45:51 AM +01 2026 ====
+ 0  0      0 169974304   3836 18186124    0    0     0     0  369  329  0  0 100  0  0
+==== Wed Feb 11 12:46:21 AM +01 2026 ====
+ 0  0      0 169975296   3836 18186672    0    0     0     0  390  326  0  0 100  0  0
+==== Wed Feb 11 12:46:52 AM +01 2026 ====
+ 0  0      0 169973872   3836 18187040    0    0     0     0  376  326  0  0 100  0  0
+==== Wed Feb 11 12:47:22 AM +01 2026 ====
+ 0  0      0 169971888   3836 18187592    0    0     0     0  426  326  0  0 100  0  0
+==== Wed Feb 11 12:47:52 AM +01 2026 ====
+ 0  0      0 169969936   3836 18187840    0    0     0     0  396  329  0  0 100  0  0
+==== Wed Feb 11 12:48:22 AM +01 2026 ====
+ 0  0      0 169966112   3836 18188516    0    0     0     0  448  328  0  0 100  0  0
+==== Wed Feb 11 12:48:52 AM +01 2026 ====
+ 0  0      0 169960048   3836 18188760    0    0     0     0  429  333  0  0 100  0  0
+==== Wed Feb 11 12:49:22 AM +01 2026 ====
+ 0  0      0 169963184   3836 18189432    0    0     0     0  473  344  0  0 100  0  0
+==== Wed Feb 11 12:49:52 AM +01 2026 ====
+ 0  0      0 169967904   3836 18189800    0    0     0     0  394  332  0  0 100  0  0
+==== Wed Feb 11 12:50:22 AM +01 2026 ====
+ 0  0      0 169945824   3836 18190272    0    0     0     0  432  317  0  0 100  0  0
+==== Wed Feb 11 12:50:52 AM +01 2026 ====
+ 0  0      0 169914624   3836 18190644    0    0     0     0  364  316  0  0 100  0  0
+==== Wed Feb 11 12:51:22 AM +01 2026 ====
+ 0  0      0 169913568   3836 18191200    0    0     0     0  431  345  0  0 100  0  0
+==== Wed Feb 11 12:51:52 AM +01 2026 ====
+ 0  0      0 169916608   3836 18191692    0    0     0     0  493  336  0  0 100  0  0
+==== Wed Feb 11 12:52:22 AM +01 2026 ====
+ 0  0      0 169906208   3836 18192120    0    0     0     0  421  310  0  0 100  0  0
+==== Wed Feb 11 12:52:52 AM +01 2026 ====
+ 0  0      0 169891184   3836 18192504    0    0     0     0  401  330  0  0 100  0  0
+==== Wed Feb 11 12:53:22 AM +01 2026 ====
+ 0  0      0 169869952   3836 18193184    0    0     0     0  403  316  0  0 100  0  0
+==== Wed Feb 11 12:53:52 AM +01 2026 ====
+ 0  0      0 169852288   3836 18193552    0    0     0     0  522  563  0  0 100  0  0
+==== Wed Feb 11 12:54:22 AM +01 2026 ====
+ 0  0      0 169846560   3836 18193984    0    0     0     0  575  566  0  0 100  0  0
+==== Wed Feb 11 12:54:52 AM +01 2026 ====
+ 0  0      0 169844720   3836 18194476    0    0     0     0  499  511  0  0 100  0  0
+==== Wed Feb 11 12:55:22 AM +01 2026 ====
+ 0  0      0 169841216   3836 18195036    0    0     0     0  562  515  0  0 100  0  0
+==== Wed Feb 11 12:55:52 AM +01 2026 ====
+ 0  0      0 169835296   3836 18195272    0    0     0     0  580  570  0  0 100  0  0
+==== Wed Feb 11 12:56:22 AM +01 2026 ====
+ 0  0      0 169837792   3836 18195948    0    0     0     0  405  316  0  0 100  0  0
+==== Wed Feb 11 12:56:52 AM +01 2026 ====
+ 0  0      0 169837632   3836 18196196    0    0     0     0  405  304  0  0 100  0  0
+==== Wed Feb 11 12:57:22 AM +01 2026 ====
+ 0  0      0 169847184   3836 18196748    0    0     0     0  445  307  0  0 100  0  0
+==== Wed Feb 11 12:57:52 AM +01 2026 ====
+ 0  0      0 169845632   3836 18197120    0    0     0     0  463  313  0  0 100  0  0
+==== Wed Feb 11 12:58:22 AM +01 2026 ====
+ 0  0      0 169844672   3836 18197668    0    0     0     0  898  786  0  0 100  0  0
+==== Wed Feb 11 12:58:52 AM +01 2026 ====
+ 0  0      0 169856144   3836 18198164    0    0     0     0  906  823  0  0 100  0  0
+==== Wed Feb 11 12:59:22 AM +01 2026 ====
+ 0  0      0 169855072   3836 18198720    0    0     0     0 1000  817  0  0 100  0  0
+==== Wed Feb 11 12:59:52 AM +01 2026 ====
+ 0  0      0 169854512   3836 18199088    0    0     0     0  895  814  0  0 100  0  0
+==== Wed Feb 11 01:00:22 AM +01 2026 ====
+ 1  0      0 169855680   3836 18199684    0    0     0     0  688  582  0  0 100  0  0
+==== Wed Feb 11 01:00:52 AM +01 2026 ====
+ 0  0      0 169857088   3836 18199932    0    0     0     0  418  319  0  0 100  0  0
+==== Wed Feb 11 01:01:22 AM +01 2026 ====
+ 0  0      0 169854592   3836 18200484    0    0     0     0  433  323  0  0 100  0  0
+==== Wed Feb 11 01:01:52 AM +01 2026 ====
+ 0  0      0 169854192   3836 18200852    0    0     0     0  421  315  0  0 100  0  0
+==== Wed Feb 11 01:02:22 AM +01 2026 ====
+ 0  0      0 169852736   3836 18201404    0    0     0     0  492  319  0  0 100  0  0
+==== Wed Feb 11 01:02:52 AM +01 2026 ====
+ 0  0      0 169851568   3836 18201772    0    0     0     0  416  314  0  0 100  0  0
+==== Wed Feb 11 01:03:22 AM +01 2026 ====
+ 0  0      0 169849584   3836 18202328    0    0     0     0  499  318  0  0 100  0  0
+==== Wed Feb 11 01:03:52 AM +01 2026 ====
+ 0  0      0 169847392   3836 18202820    0    0     0     0  626  533  0  0 100  0  0
+==== Wed Feb 11 01:04:22 AM +01 2026 ====
+ 0  0      0 169845392   3836 18203372    0    0     0     0  417  317  0  0 100  0  0
+==== Wed Feb 11 01:04:52 AM +01 2026 ====
+ 0  0      0 169844240   3836 18203616    0    0     0     0  394  315  0  0 100  0  0
+==== Wed Feb 11 01:05:22 AM +01 2026 ====
+ 0  0      0 169849312   3836 18204304    0    0     0     0  485  358  0  0 100  0  0
+==== Wed Feb 11 01:05:52 AM +01 2026 ====
+ 0  0      0 169846016   3836 18204548    0    0     0     0  404  306  0  0 100  0  0
+==== Wed Feb 11 01:06:22 AM +01 2026 ====
+ 0  0      0 169842864   3836 18205224    0    0     0     0  502  360  0  0 100  0  0
+==== Wed Feb 11 01:06:52 AM +01 2026 ====
+ 0  0      0 169841152   3836 18205592    0    0     0     0  415  329  0  0 100  0  0
+==== Wed Feb 11 01:07:22 AM +01 2026 ====
+ 0  0      0 169839136   3836 18206160    0    0     0     0  445  331  0  0 100  0  0
+==== Wed Feb 11 01:07:52 AM +01 2026 ====
+ 0  0      0 169838368   3836 18206528    0    0     0     0  437  330  0  0 100  0  0
+==== Wed Feb 11 01:08:22 AM +01 2026 ====
+ 0  0      0 169835872   3836 18207080    0    0     0     0  453  340  0  0 100  0  0
+==== Wed Feb 11 01:08:52 AM +01 2026 ====
+ 0  0      0 169838176   3836 18207324    0    0     0     0  385  301  0  0 100  0  0
+==== Wed Feb 11 01:09:22 AM +01 2026 ====
+ 0  0      0 169837392   3836 18207876    0    0     0     0  414  266  0  0 100  0  0
+==== Wed Feb 11 01:09:52 AM +01 2026 ====
+ 0  0      0 169848256   3836 18208244    0    0     0     0  310  250  0  0 100  0  0
+==== Wed Feb 11 01:10:22 AM +01 2026 ====
+ 0  0      0 169845568   3836 18208844    0    0     0     0  219  204  0  0 100  0  0
+==== Wed Feb 11 01:10:52 AM +01 2026 ====
+ 0  0      0 169843920   3836 18209212    0    0     0     0  222  241  0  0 100  0  0
+==== Wed Feb 11 01:11:22 AM +01 2026 ====
+ 0  0      0 169838032   3836 18209888    0    0     0     0  228  193  0  0 100  0  0
+==== Wed Feb 11 01:11:52 AM +01 2026 ====
+ 0  0      0 169836576   3836 18210132    0    0     0     0  308  233  0  0 100  0  0
+==== Wed Feb 11 01:12:22 AM +01 2026 ====
+ 0  0      0 169834288   3836 18210688    0    0     0     0  280  191  0  0 100  0  0
+==== Wed Feb 11 01:12:52 AM +01 2026 ====
+ 0  0      0 169833264   3836 18211056    0    0     0    11  282  260  0  0 100  0  0
+==== Wed Feb 11 01:13:22 AM +01 2026 ====
+ 1  0      0 172478432   3836 18173352    0    0     0    35 1094  312  0  1 99  0  0

performance_testing/gpu_tests_log_second_node/1gpu_output.log

@@ -0,0 +1,3 @@
+==== Job running on node: gpu03 ====
+==== Job running on node: gpu03 ====
+Cleaning up...

performance_testing/gpu_tests_log_second_node/calcul_cpu_ram_gpu_%_per_30s.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Calculates CPU%, RAM% (host), GPU_Util% and VRAM_Usage% (GPU memory used/total)
+# Expects:
+#   vmstat.txt            -> your periodic vmstat dumps (same as before)
+#   nvidia_smi.csv        -> logged via nvidia-smi --query-gpu=...,noheader,nounits
+
+VMSTAT_FILE="vmstat.txt"
+GPU_CSV="nvidia_smi.csv"
+
+if [[ ! -f "$VMSTAT_FILE" ]]; then
+  echo "File $VMSTAT_FILE not found!"
+  exit 1
+fi
+if [[ ! -f "$GPU_CSV" ]]; then
+  echo "File $GPU_CSV not found! (expected CSV from nvidia-smi --query-gpu=...)"
+  exit 1
+fi
+
+# Total host memory in KB
+TOTAL_MEM=$(grep MemTotal /proc/meminfo | awk '{print $2}')
+
+echo "CPU_Usage%,Memory_Usage%,GPU_Util%,VRAM_Usage%"
+
+# --- Load & clean vmstat lines (skip headers/blank/separators) ---
+mapfile -t vmstat_lines < <(grep -vE 'procs|r[[:space:]]+b|^=|^$' "$VMSTAT_FILE")
+
+# --- Load GPU CSV rows (timestamp,util%,mem_used(MiB),mem_total(MiB)) ---
+mapfile -t gpu_rows < <(awk 'NF{gsub(/ /,""); print}' "$GPU_CSV")
+
+len=${#vmstat_lines[@]}
+for ((i = 0; i < len; i++)); do
+  line="${vmstat_lines[$i]}"
+
+  # Extract 'free' and 'id' fields from vmstat default format
+  # vmstat columns (Linux): r b swpd free buff cache si so bi bo in cs us sy id wa st
+  read -r _ _ _ free _ _ _ _ _ _ _ _ us sy id wa st <<< "$line"
+
+  cpu_usage=$((100 - id))
+  mem_used_kb=$((TOTAL_MEM - free))
+  mem_usage_pct=$(awk -v u="$mem_used_kb" -v t="$TOTAL_MEM" 'BEGIN{printf("%.2f", (u/t)*100)}')
+
+  # Match GPU row by index; if out of range, reuse last available
+  gpu_idx=$i
+  if (( gpu_idx >= ${#gpu_rows[@]} )); then
+    gpu_idx=$((${#gpu_rows[@]} - 1))
+  fi
+  gpu_row="${gpu_rows[$gpu_idx]}"
+
+  # gpu_row format after gsub spaces: YYYY/MM/DDHH:MM:SS,UTIL,MEM_USED,MEM_TOTAL
+  IFS=',' read -r _ts util mem_used_mib mem_total_mib <<< "$gpu_row"
+
+  # Compute VRAM usage %
+  if [[ -n "$mem_used_mib" && -n "$mem_total_mib" && "$mem_total_mib" -gt 0 ]]; then
+    vram_pct=$(awk -v u="$mem_used_mib" -v t="$mem_total_mib" 'BEGIN{printf("%.2f", (u/t)*100)}')
+  else
+    vram_pct="N/A"
+  fi
+
+  # If util is empty, set N/A
+  [[ -z "$util" ]] && util="N/A"
+
+  echo "${cpu_usage}%,$mem_usage_pct%,${util}%,$vram_pct%"
+done

performance_testing/gpu_tests_log_second_node/vllm.log

@@ -0,0 +1,80 @@
+INFO 02-01 19:55:50 [__init__.py:216] Automatically detected platform cuda.
+(APIServer pid=3643986) INFO 02-01 19:56:00 [api_server.py:1839] vLLM API server version 0.11.0
+(APIServer pid=3643986) INFO 02-01 19:56:00 [utils.py:233] non-default args: {'model_tag': '/home/skiredj.abderrahman/fatnaoui/models/gptoss', 'port': 9997, 'model': '/home/skiredj.abderrahman/fatnaoui/models/gptoss'}
+(APIServer pid=3643986) WARNING 02-01 19:56:00 [__init__.py:2877] Found ulimit of 16384 and failed to automatically increase with error current limit exceeds maximum limit. This can cause fd limit errors like `OSError: [Errno 24] Too many open files`. Consider increasing with ulimit -n
+(APIServer pid=3643986) INFO 02-01 19:56:09 [model.py:547] Resolved architecture: GptOssForCausalLM
+(APIServer pid=3643986) `torch_dtype` is deprecated! Use `dtype` instead!
+(APIServer pid=3643986) ERROR 02-01 19:56:09 [config.py:278] Error retrieving safetensors: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/home/skiredj.abderrahman/fatnaoui/models/gptoss'. Use `repo_type` argument if needed., retrying 1 of 2
+(APIServer pid=3643986) ERROR 02-01 19:56:11 [config.py:276] Error retrieving safetensors: Repo id must be in the form 'repo_name' or 'namespace/repo_name': '/home/skiredj.abderrahman/fatnaoui/models/gptoss'. Use `repo_type` argument if needed.
+(APIServer pid=3643986) INFO 02-01 19:56:11 [model.py:1730] Downcasting torch.float32 to torch.bfloat16.
+(APIServer pid=3643986) INFO 02-01 19:56:11 [model.py:1510] Using max model len 131072
+(APIServer pid=3643986) INFO 02-01 19:56:13 [scheduler.py:205] Chunked prefill is enabled with max_num_batched_tokens=2048.
+(APIServer pid=3643986) INFO 02-01 19:56:13 [config.py:271] Overriding max cuda graph capture size to 992 for performance.
+INFO 02-01 19:56:17 [__init__.py:216] Automatically detected platform cuda.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:23 [core.py:644] Waiting for init message from front-end.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:23 [core.py:77] Initializing a V1 LLM engine (v0.11.0) with config: model='/home/skiredj.abderrahman/fatnaoui/models/gptoss', speculative_config=None, tokenizer='/home/skiredj.abderrahman/fatnaoui/models/gptoss', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=131072, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, disable_custom_all_reduce=False, quantization=mxfp4, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_fallback=False, disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='openai_gptoss'), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None), seed=0, served_model_name=/home/skiredj.abderrahman/fatnaoui/models/gptoss, enable_prefix_caching=True, chunked_prefill_enabled=True, pooler_config=None, compilation_config={"level":3,"debug_dump_path":"","cache_dir":"","backend":"","custom_ops":[],"splitting_ops":["vllm.unified_attention","vllm.unified_attention_with_output","vllm.mamba_mixer2","vllm.mamba_mixer","vllm.short_conv","vllm.linear_attention","vllm.plamo2_mamba_mixer","vllm.gdn_attention","vllm.sparse_attn_indexer"],"use_inductor":true,"compile_sizes":[],"inductor_compile_config":{"enable_auto_functionalized_v2":false},"inductor_passes":{},"cudagraph_mode":[2,1],"use_cudagraph":true,"cudagraph_num_of_warmups":1,"cudagraph_capture_sizes":[992,976,960,944,928,912,896,880,864,848,832,816,800,784,768,752,736,720,704,688,672,656,640,624,608,592,576,560,544,528,512,496,480,464,448,432,416,400,384,368,352,336,320,304,288,272,256,248,240,232,224,216,208,200,192,184,176,168,160,152,144,136,128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],"cudagraph_copy_inputs":false,"full_cuda_graph":false,"use_inductor_graph_partition":false,"pass_config":{},"max_capture_size":992,"local_cache_dir":null}
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:24 [parallel_state.py:1208] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0, EP rank 0
+(EngineCore_DP0 pid=3644175) WARNING 02-01 19:56:24 [topk_topp_sampler.py:66] FlashInfer is not available. Falling back to the PyTorch-native implementation of top-p & top-k sampling. For the best performance, please install FlashInfer.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:24 [gpu_model_runner.py:2602] Starting to load model /home/skiredj.abderrahman/fatnaoui/models/gptoss...
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:24 [gpu_model_runner.py:2634] Loading model from scratch...
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:24 [cuda.py:361] Using Triton backend on V1 engine.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:24 [mxfp4.py:98] Using Marlin backend
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:43 [default_loader.py:267] Loading weights took 18.56 seconds
+(EngineCore_DP0 pid=3644175) WARNING 02-01 19:56:43 [marlin_utils_fp4.py:196] Your GPU does not have native support for FP4 computation but FP4 quantization is being used. Weight-only FP4 compression will be used leveraging the Marlin kernel. This may degrade performance for compute-heavy workloads.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:44 [gpu_model_runner.py:2653] Model loading took 13.7194 GiB and 19.054449 seconds
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:49 [backends.py:548] Using cache directory: /home/skiredj.abderrahman/.cache/vllm/torch_compile_cache/53365dd68b/rank_0_0/backbone for vLLM's torch.compile
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:49 [backends.py:559] Dynamo bytecode transform time: 5.05 s
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:56:50 [backends.py:197] Cache the graph for dynamic shape for later use
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:08 [backends.py:218] Compiling a graph for dynamic shape takes 18.54 s
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:08 [marlin_utils.py:353] You are running Marlin kernel with bf16 on GPUs before SM90. You can consider change to fp16 to achieve better performance if possible.
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:09 [monitor.py:34] torch.compile takes 23.60 s in total
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:10 [gpu_worker.py:298] Available KV cache memory: 24.47 GiB
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:10 [kv_cache_utils.py:1087] GPU KV cache size: 534,528 tokens
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:10 [kv_cache_utils.py:1091] Maximum concurrency for 131,072 tokens per request: 8.02x
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) 
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:20 [gpu_model_runner.py:3480] Graph capturing finished in 9 secs, took 0.95 GiB
+(EngineCore_DP0 pid=3644175) INFO 02-01 19:57:20 [core.py:210] init engine (profile, create kv cache, warmup model) took 35.99 seconds
+(APIServer pid=3643986) INFO 02-01 19:57:21 [loggers.py:147] Engine 000: vllm cache_config_info with initialization after num_gpu_blocks is: 66817
+(APIServer pid=3643986) INFO 02-01 19:57:21 [api_server.py:1634] Supported_tasks: ['generate']
+(APIServer pid=3643986) WARNING 02-01 19:57:21 [serving_responses.py:154] For gpt-oss, we ignore --enable-auto-tool-choice and always enable tool use.
+[rank0]:[W201 19:57:26.951036535 ProcessGroupNCCL.cpp:1538] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+(APIServer pid=3643986) Traceback (most recent call last):
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/bin/vllm", line 7, in <module>
+(APIServer pid=3643986)     sys.exit(main())
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/cli/main.py", line 54, in main
+(APIServer pid=3643986)     args.dispatch_function(args)
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/cli/serve.py", line 57, in cmd
+(APIServer pid=3643986)     uvloop.run(run_server(args))
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/uvloop/__init__.py", line 69, in run
+(APIServer pid=3643986)     return loop.run_until_complete(wrapper())
+(APIServer pid=3643986)   File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/uvloop/__init__.py", line 48, in wrapper
+(APIServer pid=3643986)     return await main
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 1884, in run_server
+(APIServer pid=3643986)     await run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 1910, in run_server_worker
+(APIServer pid=3643986)     await init_app_state(engine_client, vllm_config, app.state, args)
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/api_server.py", line 1694, in init_app_state
+(APIServer pid=3643986)     state.openai_serving_responses = OpenAIServingResponses(
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/openai/serving_responses.py", line 161, in __init__
+(APIServer pid=3643986)     get_stop_tokens_for_assistant_actions())
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/harmony_utils.py", line 403, in get_stop_tokens_for_assistant_actions
+(APIServer pid=3643986)     return get_encoding().stop_tokens_for_assistant_actions()
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/vllm/entrypoints/harmony_utils.py", line 56, in get_encoding
+(APIServer pid=3643986)     _harmony_encoding = load_harmony_encoding(
+(APIServer pid=3643986)   File "/home/skiredj.abderrahman/.conda/envs/gptoss-vllm/lib/python3.10/site-packages/openai_harmony/__init__.py", line 689, in load_harmony_encoding
+(APIServer pid=3643986)     inner: _PyHarmonyEncoding = _load_harmony_encoding(name)
+(APIServer pid=3643986) openai_harmony.HarmonyError: error downloading or loading vocab file: failed to download or load vocab file

performance_testing/run_model_1.pbs

@@ -0,0 +1,19 @@
+#!/bin/bash
+#PBS -N 1gpu_test
+#PBS -l select=1:ncpus=20:mem=180gb:ngpus=1
+#PBS -q gpu_1d
+#PBS -o logs/output.log
+#PBS -e logs/error.log
+
+# === Load modules & activate conda ===
+module use /app/common/modules
+module load anaconda3-2024.10
+source activate gptoss20b_khalil
+
+
+# === Confirm node ===
+echo "==== Job running on node: $(hostname -s) ===="
+
+# === Start vLLM server ===
+vllm serve /home/skiredj.abderrahman/fatnaoui/models/gptoss --port 9998
+