matCercola18 commited on
Commit
e91d8e7
·
verified ·
1 Parent(s): dfae7d7

Backup of this project's launch records from andromeda (1/1)

Browse files
andromeda_home/icml_r3_X1S_launch.sh ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # usage: icml_r3_X1S_launch.sh LOG cmd... (hard timeout ends 08:20 PT; records pid)
3
+ LOG=$1; shift
4
+ T=$(( $(TZ=America/Los_Angeles date -d "today 08:20" +%s) - $(date +%s) ))
5
+ cd ~/DiffusionControl; export WANDB_MODE=offline OMP_NUM_THREADS=1 RAYON_NUM_THREADS=1 PYTHONPATH=.
6
+ setsid nohup timeout --signal=TERM $T "$@" > $LOG 2>&1 < /dev/null & disown
7
+ sleep 1; echo "$(date +%F_%T) launched $LOG: $*" >> ~/icml_r3_pids/X1S.txt
andromeda_home/icml_r3_pids/X1F.txt ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 3272228 flow_s0 train 2026-10-13 00:25:51.620948-07:00
2
+ 3272223
3
+ 3272548 mip_s0 train 2026-10-13 00:26:21.638435-07:00
4
+ 3272805 mdn_M4_s0 train 2026-10-13 00:26:51.656421-07:00
5
+ 3273658 mdn_M8_s0 train 2026-10-13 00:27:21.676173-07:00
6
+ 3298029 wta_M16_s0 train 2026-10-13 01:11:51.739408-07:00
7
+ 3301672 cvae_z4_b1e-1_s0 train 2026-10-13 01:19:01.770401-07:00
8
+ 3302489 cvae_z16_b1e-2_s0 train 2026-10-13 01:20:51.794568-07:00
9
+ 3307387 wta_M32_s0 train 2026-10-13 01:30:21.820418-07:00
10
+ 3328590 diff_s0 train 2026-10-13 02:19:11.871761-07:00
11
+ 3333235 flow_s1 train 2026-10-13 02:30:01.895972-07:00
12
+ 3333943 flow_s2 train 2026-10-13 02:31:31.911175-07:00
13
+ 3335684 mip_s1 train 2026-10-13 02:38:41.931454-07:00
14
+ 3352261 mip_s2 train 2026-10-13 03:28:11.989494-07:00
15
+ 3354121 mdn_M4_s1 train 2026-10-13 03:38:22.012702-07:00
16
+ 3354450 mdn_M4_s2 train 2026-10-13 03:40:12.028702-07:00
17
+ 3359721 cvae_z16_b1e-2_s1 train 2026-10-13 04:03:22.062899-07:00
18
+ 3367883 cvae_z16_b1e-2_s2 train 2026-10-13 04:45:32.113857-07:00
19
+ 3368439 wta_M32_s1 train 2026-10-13 04:47:02.129204-07:00
20
+ 3369282 wta_M32_s2 train 2026-10-13 04:48:52.145547-07:00
andromeda_home/icml_r3_pids/X1S.txt ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ 2026-10-13_00:27:09 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_A.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 11 results/icml_r3/X1S/jobs_grid_m4_a.json results/icml_r3/X1S/jobs_grid_m3_a.json
2
+ 2026-10-13_00:27:10 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_B.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 12 results/icml_r3/X1S/jobs_grid_m4_b.json results/icml_r3/X1S/jobs_grid_m3_b.json
3
+ 2026-10-13_00:28:29 pids: timeout 3273061 3273071; python 3273064 3273074 (grid m4 a/b -> m3 a/b)
4
+ 2026-10-13_00:33:36 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_C.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 13 results/icml_r3/X1S/jobs_x3par.json
5
+ 2026-10-13_00:33:44 pids: timeout 3276904; python 3276907 (x3 parity m3 diff/reg s0)
6
+ 2026-10-13_00:53:12 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_D.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 14 results/icml_r3/X1S/jobs_x3_s3_a.json results/icml_r3/X1S/jobs_x3_s3_b.json results/icml_r3/X1S/jobs_x3_s4_a.json results/icml_r3/X1S/jobs_x3_s4_b.json
7
+ 2026-10-13_00:53:44 launched chain D (x3 seeds 3,4 batches of 4)
8
+ 3286291 3286293 3286949 2026-10-13_02:35:09 launched /home/cercola/DiffusionControl/results/icml_r3/X1S/logs/chain_E.log: /home/cercola/DiffusionControl/results/icml_r3/X1S/chain_grid.sh 15 results/icml_r3/X1S/jobs_cvae_ext_m4.json
9
+ 2026-10-13_02:35:20 launched chain E (m4 cvae ext); pids 3334905 3334907 3335029
10
+ 3338264 3338267 3338269 <- queue_runner pids
11
+ 2026-10-13_02:44:56 virgo driver.py launched (pids 1200490 1200495 1200496 1200617 1200620 )
12
+ 2026-10-13_02:52:17 queue_runner launched 10_H1_m3cvae_m4flow_s12 (est 3050 MiB, mine 4358, free 20700)
13
+ 2026-10-13_03:04:40 virgo driver.py RESTARTED (staged m=3 selection); pids 1209818 1209824 1209825 1209846 1209849
14
+ 2026-10-13_03:07:29 virgo driver restarted (fill_doc)
15
+ 2026-10-13_03:53:36 queue_runner launched 20_H2_m4_mdn_wta_mip_tsd_s12 (est 2550 MiB, mine 4080, free 20895)
16
+ 2026-10-13_04:03:51 queue_runner launched 25_m3_noncvae_s12 (est 3050 MiB, mine 4752, free 20347)
17
+ 2026-10-13_04:47:09 queue_runner launched 30_m4_cvae_s12 (est 1550 MiB, mine 5106, free 19314)
18
+ 2026-10-13_05:21:26 queue_runner launched 41_m3_cvae_s12 (est 1550 MiB, mine 4138, free 21809)
andromeda_home/icml_r3_pids/X2.txt ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ 3269402
2
+ bank_parent 3269402 (24 forked workers children)
3
+ post_bank 3272394
4
+ post_bank(v2, replaces 3272394) 3278281
5
+ train_queue 3286094
6
+ killed queue parent+xargs 3286094 3286098 at Tue Oct 13 01:12:44 AM PDT 2026 so no extra jobs auto-start; 4 trainings continue (timeouts 08:20)
7
+ val_misc 3298888
8
+ stageA 3303394
9
+ driver(andromeda, timeout 08:20) pid 3342161 started Tue Oct 13 02:52:11 AM PDT 2026
andromeda_home/icml_r3_pids/hf_upload.log ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ OK models/icml_r3/resume/x1f
2
+ OK models/icml_r3/resume/x2
andromeda_home/icml_r3_pids/watchdog.log ADDED
@@ -0,0 +1 @@
 
 
1
+ Tue Oct 13 07:01:07 AM PDT 2026 orchestrator stopped idle queue_runner 3338267/3338269 and watchdog 3267996 (all work done)
andromeda_home/icml_r3_pids/watchdog.sh ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # At 08:25 PT stop only the GPU jobs this session recorded in ~/icml_r3_pids/*.txt (own user, python/timeout only).
3
+ T=$(( $(TZ=America/Los_Angeles date -d "today 08:25" +%s) - $(date +%s) )); [ $T -gt 0 ] && sleep $T
4
+ for f in ~/icml_r3_pids/*.txt; do [ -f "$f" ] || continue
5
+ while read -r pid _; do [[ "$pid" =~ ^[0-9]+$ ]] || continue
6
+ [ "$(ps -o user= -p $pid 2>/dev/null)" = "cercola" ] || continue
7
+ c=$(ps -o comm= -p $pid); [[ "$c" == python* || "$c" == timeout ]] || continue
8
+ echo "$(date) TERM $pid $c ($f)"; kill -TERM $pid
9
+ done < "$f"; done
10
+ sleep 60; nvidia-smi --query-compute-apps=pid,used_memory,process_name --format=csv,noheader