Backup of this project's launch records from andromeda (1/1)
Browse files- andromeda_home/icml_r3_X1S_launch.sh +7 -0
- andromeda_home/icml_r3_pids/X1F.txt +20 -0
- andromeda_home/icml_r3_pids/X1S.txt +18 -0
- andromeda_home/icml_r3_pids/X2.txt +9 -0
- andromeda_home/icml_r3_pids/hf_upload.log +2 -0
- andromeda_home/icml_r3_pids/watchdog.log +1 -0
- andromeda_home/icml_r3_pids/watchdog.sh +10 -0
andromeda_home/icml_r3_X1S_launch.sh
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# usage: icml_r3_X1S_launch.sh LOG cmd... (hard timeout ends 08:20 PT; records pid)
|
| 3 |
+
LOG=$1; shift
|
| 4 |
+
T=$(( $(TZ=America/Los_Angeles date -d "today 08:20" +%s) - $(date +%s) ))
|
| 5 |
+
cd ~/DiffusionControl; export WANDB_MODE=offline OMP_NUM_THREADS=1 RAYON_NUM_THREADS=1 PYTHONPATH=.
|
| 6 |
+
setsid nohup timeout --signal=TERM $T "$@" > $LOG 2>&1 < /dev/null & disown
|
| 7 |
+
sleep 1; echo "$(date +%F_%T) launched $LOG: $*" >> ~/icml_r3_pids/X1S.txt
|
andromeda_home/icml_r3_pids/X1F.txt
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
3272228 flow_s0 train 2026-10-13 00:25:51.620948-07:00
|
| 2 |
+
3272223
|
| 3 |
+
3272548 mip_s0 train 2026-10-13 00:26:21.638435-07:00
|
| 4 |
+
3272805 mdn_M4_s0 train 2026-10-13 00:26:51.656421-07:00
|
| 5 |
+
3273658 mdn_M8_s0 train 2026-10-13 00:27:21.676173-07:00
|
| 6 |
+
3298029 wta_M16_s0 train 2026-10-13 01:11:51.739408-07:00
|
| 7 |
+
3301672 cvae_z4_b1e-1_s0 train 2026-10-13 01:19:01.770401-07:00
|
| 8 |
+
3302489 cvae_z16_b1e-2_s0 train 2026-10-13 01:20:51.794568-07:00
|
| 9 |
+
3307387 wta_M32_s0 train 2026-10-13 01:30:21.820418-07:00
|
| 10 |
+
3328590 diff_s0 train 2026-10-13 02:19:11.871761-07:00
|
| 11 |
+
3333235 flow_s1 train 2026-10-13 02:30:01.895972-07:00
|
| 12 |
+
3333943 flow_s2 train 2026-10-13 02:31:31.911175-07:00
|
| 13 |
+
3335684 mip_s1 train 2026-10-13 02:38:41.931454-07:00
|
| 14 |
+
3352261 mip_s2 train 2026-10-13 03:28:11.989494-07:00
|
| 15 |
+
3354121 mdn_M4_s1 train 2026-10-13 03:38:22.012702-07:00
|
| 16 |
+
3354450 mdn_M4_s2 train 2026-10-13 03:40:12.028702-07:00
|
| 17 |
+
3359721 cvae_z16_b1e-2_s1 train 2026-10-13 04:03:22.062899-07:00
|
| 18 |
+
3367883 cvae_z16_b1e-2_s2 train 2026-10-13 04:45:32.113857-07:00
|
| 19 |
+
3368439 wta_M32_s1 train 2026-10-13 04:47:02.129204-07:00
|
| 20 |
+
3369282 wta_M32_s2 train 2026-10-13 04:48:52.145547-07:00
|
andromeda_home/icml_r3_pids/X1S.txt
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
2026-10-13_00:27:09 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_A.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 11 results/icml_r3/X1S/jobs_grid_m4_a.json results/icml_r3/X1S/jobs_grid_m3_a.json
|
| 2 |
+
2026-10-13_00:27:10 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_B.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 12 results/icml_r3/X1S/jobs_grid_m4_b.json results/icml_r3/X1S/jobs_grid_m3_b.json
|
| 3 |
+
2026-10-13_00:28:29 pids: timeout 3273061 3273071; python 3273064 3273074 (grid m4 a/b -> m3 a/b)
|
| 4 |
+
2026-10-13_00:33:36 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_C.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 13 results/icml_r3/X1S/jobs_x3par.json
|
| 5 |
+
2026-10-13_00:33:44 pids: timeout 3276904; python 3276907 (x3 parity m3 diff/reg s0)
|
| 6 |
+
2026-10-13_00:53:12 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_D.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 14 results/icml_r3/X1S/jobs_x3_s3_a.json results/icml_r3/X1S/jobs_x3_s3_b.json results/icml_r3/X1S/jobs_x3_s4_a.json results/icml_r3/X1S/jobs_x3_s4_b.json
|
| 7 |
+
2026-10-13_00:53:44 launched chain D (x3 seeds 3,4 batches of 4)
|
| 8 |
+
3286291 3286293 3286949 2026-10-13_02:35:09 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_E.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 15 results/icml_r3/X1S/jobs_cvae_ext_m4.json
|
| 9 |
+
2026-10-13_02:35:20 launched chain E (m4 cvae ext); pids 3334905 3334907 3335029
|
| 10 |
+
3338264 3338267 3338269 <- queue_runner pids
|
| 11 |
+
2026-10-13_02:44:56 virgo driver.py launched (pids 1200490 1200495 1200496 1200617 1200620 )
|
| 12 |
+
2026-10-13_02:52:17 queue_runner launched 10_H1_m3cvae_m4flow_s12 (est 3050 MiB, mine 4358, free 20700)
|
| 13 |
+
2026-10-13_03:04:40 virgo driver.py RESTARTED (staged m=3 selection); pids 1209818 1209824 1209825 1209846 1209849
|
| 14 |
+
2026-10-13_03:07:29 virgo driver restarted (fill_doc)
|
| 15 |
+
2026-10-13_03:53:36 queue_runner launched 20_H2_m4_mdn_wta_mip_tsd_s12 (est 2550 MiB, mine 4080, free 20895)
|
| 16 |
+
2026-10-13_04:03:51 queue_runner launched 25_m3_noncvae_s12 (est 3050 MiB, mine 4752, free 20347)
|
| 17 |
+
2026-10-13_04:47:09 queue_runner launched 30_m4_cvae_s12 (est 1550 MiB, mine 5106, free 19314)
|
| 18 |
+
2026-10-13_05:21:26 queue_runner launched 41_m3_cvae_s12 (est 1550 MiB, mine 4138, free 21809)
|
andromeda_home/icml_r3_pids/X2.txt
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
3269402
|
| 2 |
+
bank_parent 3269402 (24 forked workers children)
|
| 3 |
+
post_bank 3272394
|
| 4 |
+
post_bank(v2, replaces 3272394) 3278281
|
| 5 |
+
train_queue 3286094
|
| 6 |
+
killed queue parent+xargs 3286094 3286098 at Tue Oct 13 01:12:44 AM PDT 2026 so no extra jobs auto-start; 4 trainings continue (timeouts 08:20)
|
| 7 |
+
val_misc 3298888
|
| 8 |
+
stageA 3303394
|
| 9 |
+
driver(andromeda, timeout 08:20) pid 3342161 started Tue Oct 13 02:52:11 AM PDT 2026
|
andromeda_home/icml_r3_pids/hf_upload.log
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
OK models/icml_r3/resume/x1f
|
| 2 |
+
OK models/icml_r3/resume/x2
|
andromeda_home/icml_r3_pids/watchdog.log
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
Tue Oct 13 07:01:07 AM PDT 2026 orchestrator stopped idle queue_runner 3338267/3338269 and watchdog 3267996 (all work done)
|
andromeda_home/icml_r3_pids/watchdog.sh
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
# At 08:25 PT stop only the GPU jobs this session recorded in ~/icml_r3_pids/*.txt (own user, python/timeout only).
|
| 3 |
+
T=$(( $(TZ=America/Los_Angeles date -d "today 08:25" +%s) - $(date +%s) )); [ $T -gt 0 ] && sleep $T
|
| 4 |
+
for f in ~/icml_r3_pids/*.txt; do [ -f "$f" ] || continue
|
| 5 |
+
while read -r pid _; do [[ "$pid" =~ ^[0-9]+$ ]] || continue
|
| 6 |
+
[ "$(ps -o user= -p $pid 2>/dev/null)" = "cercola" ] || continue
|
| 7 |
+
c=$(ps -o comm= -p $pid); [[ "$c" == python* || "$c" == timeout ]] || continue
|
| 8 |
+
echo "$(date) TERM $pid $c ($f)"; kill -TERM $pid
|
| 9 |
+
done < "$f"; done
|
| 10 |
+
sleep 60; nvidia-smi --query-compute-apps=pid,used_memory,process_name --format=csv,noheader
|