eval/mauve_temp10_lambda_grid_8gpu/driver.log

[2026-05-05 12:21:26] starting temp=1.0 lambda grid on GPUs: 0,1,2,3,4,5,6,7
[2026-05-05 12:21:26] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp10_lambda_grid_8gpu
[2026-05-05 12:21:26] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-05 12:21:26] models=top1,top2,top3,h2,h3
[2026-05-05 12:21:26] launched top1_pps1_lam1.0_temp1.0 on gpu0 pid=3725829
[2026-05-05 12:21:26] launched top2_pps1_lam1.0_temp1.0 on gpu1 pid=3725834
[2026-05-05 12:21:26] launched top3_pps1_lam1.0_temp1.0 on gpu2 pid=3725839
[2026-05-05 12:21:26] launched h2_pps1_lam1.0_temp1.0 on gpu3 pid=3725844
[2026-05-05 12:21:26] launched h3_pps1_lam1.0_temp1.0 on gpu4 pid=3725849
[2026-05-05 12:21:26] launched top1_pps1_lam0.8_temp1.0 on gpu5 pid=3725854
[2026-05-05 12:21:26] launched top2_pps1_lam0.8_temp1.0 on gpu6 pid=3725859
[2026-05-05 12:21:26] launched top3_pps1_lam0.8_temp1.0 on gpu7 pid=3725864
[2026-05-05 12:21:41] failed h2_pps1_lam1.0_temp1.0 on gpu3 pid=3725844 exit=-1
[2026-05-05 12:21:41] launched h2_pps1_lam0.8_temp1.0 on gpu3 pid=3728669
[2026-05-05 12:21:41] failed h3_pps1_lam1.0_temp1.0 on gpu4 pid=3725849 exit=1
[2026-05-05 12:21:41] launched h3_pps1_lam0.8_temp1.0 on gpu4 pid=3728677
[2026-05-05 12:21:51] failed h2_pps1_lam0.8_temp1.0 on gpu3 pid=3728669 exit=-1
[2026-05-05 12:21:51] launched top1_pps1_lam0.6_temp1.0 on gpu3 pid=3729452
[2026-05-05 12:21:56] failed h3_pps1_lam0.8_temp1.0 on gpu4 pid=3728677 exit=-1
[2026-05-05 12:21:56] launched top2_pps1_lam0.6_temp1.0 on gpu4 pid=3729717
[2026-05-05 12:27:32] nonzero exit (-1) for top2_pps1_lam0.8_temp1.0 on gpu6 pid=3725859 but output exists
[2026-05-05 12:27:32] launched top3_pps1_lam0.6_temp1.0 on gpu6 pid=3739605
[2026-05-05 12:28:27] nonzero exit (-1) for top2_pps1_lam0.6_temp1.0 on gpu4 pid=3729717 but output exists
[2026-05-05 12:28:27] launched h2_pps1_lam0.6_temp1.0 on gpu4 pid=3740806
[2026-05-05 12:28:47] failed h2_pps1_lam0.6_temp1.0 on gpu4 pid=3740806 exit=-1
[2026-05-05 12:28:47] launched h3_pps1_lam0.6_temp1.0 on gpu4 pid=3741492
[2026-05-05 12:29:07] failed h3_pps1_lam0.6_temp1.0 on gpu4 pid=3741492 exit=-1
[2026-05-05 12:29:07] launched top1_pps1_lam0.4_temp1.0 on gpu4 pid=3741989
[2026-05-05 12:29:12] nonzero exit (-1) for top2_pps1_lam1.0_temp1.0 on gpu1 pid=3725834 but output exists
[2026-05-05 12:29:12] launched top2_pps1_lam0.4_temp1.0 on gpu1 pid=3742170
[2026-05-05 12:29:32] nonzero exit (-1) for top1_pps1_lam0.6_temp1.0 on gpu3 pid=3729452 but output exists
[2026-05-05 12:29:32] launched top3_pps1_lam0.4_temp1.0 on gpu3 pid=3742894
[2026-05-05 12:29:57] nonzero exit (-1) for top1_pps1_lam0.8_temp1.0 on gpu5 pid=3725854 but output exists
[2026-05-05 12:29:57] launched h2_pps1_lam0.4_temp1.0 on gpu5 pid=3743507
[2026-05-05 12:30:12] nonzero exit (-1) for top1_pps1_lam1.0_temp1.0 on gpu0 pid=3725829 but output exists
[2026-05-05 12:30:12] launched h3_pps1_lam0.4_temp1.0 on gpu0 pid=3744010
[2026-05-05 12:30:12] finished top3_pps1_lam1.0_temp1.0 on gpu2 pid=3725839
[2026-05-05 12:30:12] launched top1_pps1_lam0.2_temp1.0 on gpu2 pid=3744018
[2026-05-05 12:30:12] failed h2_pps1_lam0.4_temp1.0 on gpu5 pid=3743507 exit=1
[2026-05-05 12:30:12] launched top2_pps1_lam0.2_temp1.0 on gpu5 pid=3744026
[2026-05-05 12:30:12] finished top3_pps1_lam0.8_temp1.0 on gpu7 pid=3725864
[2026-05-05 12:30:12] launched top3_pps1_lam0.2_temp1.0 on gpu7 pid=3744034
[2026-05-05 12:30:27] failed h3_pps1_lam0.4_temp1.0 on gpu0 pid=3744010 exit=-1
[2026-05-05 12:30:27] launched h2_pps1_lam0.2_temp1.0 on gpu0 pid=3745540
[2026-05-05 12:30:42] failed h2_pps1_lam0.2_temp1.0 on gpu0 pid=3745540 exit=-1
[2026-05-05 12:30:42] launched h3_pps1_lam0.2_temp1.0 on gpu0 pid=3746052
[2026-05-05 12:30:57] failed h3_pps1_lam0.2_temp1.0 on gpu0 pid=3746052 exit=-1
[2026-05-05 12:30:57] launched top1_pps2_lam1.0_temp1.0 on gpu0 pid=3746543
[2026-05-05 12:34:52] nonzero exit (-1) for top3_pps1_lam0.6_temp1.0 on gpu6 pid=3739605 but output exists
[2026-05-05 12:34:52] launched top2_pps2_lam1.0_temp1.0 on gpu6 pid=3755471
[2026-05-05 12:35:02] nonzero exit (-1) for top2_pps1_lam0.4_temp1.0 on gpu1 pid=3742170 but output exists
[2026-05-05 12:35:02] launched top3_pps2_lam1.0_temp1.0 on gpu1 pid=3755942
[2026-05-05 12:35:17] nonzero exit (-1) for top1_pps1_lam0.4_temp1.0 on gpu4 pid=3741989 but output exists
[2026-05-05 12:35:17] launched h2_pps2_lam1.0_temp1.0 on gpu4 pid=3756416
[2026-05-05 12:35:32] failed h2_pps2_lam1.0_temp1.0 on gpu4 pid=3756416 exit=-1
[2026-05-05 12:35:32] launched h3_pps2_lam1.0_temp1.0 on gpu4 pid=3756762
[2026-05-05 12:35:42] nonzero exit (-1) for top2_pps1_lam0.2_temp1.0 on gpu5 pid=3744026 but output exists
[2026-05-05 12:35:42] launched top1_pps2_lam0.8_temp1.0 on gpu5 pid=3757107
[2026-05-05 12:35:47] nonzero exit (-1) for top1_pps1_lam0.2_temp1.0 on gpu2 pid=3744018 but output exists
[2026-05-05 12:35:47] launched top2_pps2_lam0.8_temp1.0 on gpu2 pid=3757379
[2026-05-05 12:35:47] failed h3_pps2_lam1.0_temp1.0 on gpu4 pid=3756762 exit=1
[2026-05-05 12:35:47] launched top3_pps2_lam0.8_temp1.0 on gpu4 pid=3757387
[2026-05-05 12:36:47] nonzero exit (-1) for top1_pps2_lam1.0_temp1.0 on gpu0 pid=3746543 but output exists
[2026-05-05 12:36:47] launched h2_pps2_lam0.8_temp1.0 on gpu0 pid=3759110
[2026-05-05 12:36:52] nonzero exit (-1) for top3_pps1_lam0.4_temp1.0 on gpu3 pid=3742894 but output exists
[2026-05-05 12:36:52] launched h3_pps2_lam0.8_temp1.0 on gpu3 pid=3759316
[2026-05-05 12:36:52] finished top3_pps1_lam0.2_temp1.0 on gpu7 pid=3744034
[2026-05-05 12:36:52] launched top1_pps2_lam0.6_temp1.0 on gpu7 pid=3759329
[2026-05-05 12:37:02] failed h2_pps2_lam0.8_temp1.0 on gpu0 pid=3759110 exit=-1
[2026-05-05 12:37:02] launched top2_pps2_lam0.6_temp1.0 on gpu0 pid=3760131
[2026-05-05 12:37:07] failed h3_pps2_lam0.8_temp1.0 on gpu3 pid=3759316 exit=-1
[2026-05-05 12:37:07] launched top3_pps2_lam0.6_temp1.0 on gpu3 pid=3760277
[2026-05-05 12:39:52] nonzero exit (-1) for top2_pps2_lam1.0_temp1.0 on gpu6 pid=3755471 but output exists
[2026-05-05 12:39:52] launched h2_pps2_lam0.6_temp1.0 on gpu6 pid=3766116
[2026-05-05 12:40:07] failed h2_pps2_lam0.6_temp1.0 on gpu6 pid=3766116 exit=-1
[2026-05-05 12:40:07] launched h3_pps2_lam0.6_temp1.0 on gpu6 pid=3766589
[2026-05-05 12:40:22] failed h3_pps2_lam0.6_temp1.0 on gpu6 pid=3766589 exit=-1
[2026-05-05 12:40:22] launched top1_pps2_lam0.4_temp1.0 on gpu6 pid=3767196
[2026-05-05 12:40:32] nonzero exit (-1) for top3_pps2_lam1.0_temp1.0 on gpu1 pid=3755942 but output exists
[2026-05-05 12:40:32] launched top2_pps2_lam0.4_temp1.0 on gpu1 pid=3767783
[2026-05-05 12:40:52] nonzero exit (-1) for top2_pps2_lam0.8_temp1.0 on gpu2 pid=3757379 but output exists
[2026-05-05 12:40:52] launched top3_pps2_lam0.4_temp1.0 on gpu2 pid=3768268
[2026-05-05 12:41:07] nonzero exit (-1) for top1_pps2_lam0.8_temp1.0 on gpu5 pid=3757107 but output exists
[2026-05-05 12:41:07] launched h2_pps2_lam0.4_temp1.0 on gpu5 pid=3768615
[2026-05-05 12:41:17] nonzero exit (-1) for top3_pps2_lam0.8_temp1.0 on gpu4 pid=3757387 but output exists
[2026-05-05 12:41:17] launched h3_pps2_lam0.4_temp1.0 on gpu4 pid=3769239
[2026-05-05 12:41:22] failed h2_pps2_lam0.4_temp1.0 on gpu5 pid=3768615 exit=-1
[2026-05-05 12:41:22] launched top1_pps2_lam0.2_temp1.0 on gpu5 pid=3769555
[2026-05-05 12:41:32] failed h3_pps2_lam0.4_temp1.0 on gpu4 pid=3769239 exit=-1
[2026-05-05 12:41:32] launched top2_pps2_lam0.2_temp1.0 on gpu4 pid=3770095
[2026-05-05 12:42:02] nonzero exit (-1) for top2_pps2_lam0.6_temp1.0 on gpu0 pid=3760131 but output exists
[2026-05-05 12:42:02] launched top3_pps2_lam0.2_temp1.0 on gpu0 pid=3771923
[2026-05-05 12:42:07] nonzero exit (-1) for top1_pps2_lam0.6_temp1.0 on gpu7 pid=3759329 but output exists
[2026-05-05 12:42:07] launched h2_pps2_lam0.2_temp1.0 on gpu7 pid=3772067
[2026-05-05 12:42:22] failed h2_pps2_lam0.2_temp1.0 on gpu7 pid=3772067 exit=-1
[2026-05-05 12:42:22] launched h3_pps2_lam0.2_temp1.0 on gpu7 pid=3772609
[2026-05-05 12:42:32] nonzero exit (-1) for top3_pps2_lam0.6_temp1.0 on gpu3 pid=3760277 but output exists
[2026-05-05 12:42:32] launched top1_pps4_lam1.0_temp1.0 on gpu3 pid=3772953
[2026-05-05 12:42:37] failed h3_pps2_lam0.2_temp1.0 on gpu7 pid=3772609 exit=-1
[2026-05-05 12:42:37] launched top2_pps4_lam1.0_temp1.0 on gpu7 pid=3773141
[2026-05-05 12:45:08] nonzero exit (-1) for top1_pps2_lam0.4_temp1.0 on gpu6 pid=3767196 but output exists
[2026-05-05 12:45:08] launched top3_pps4_lam1.0_temp1.0 on gpu6 pid=3778515
[2026-05-05 12:45:13] nonzero exit (-1) for top2_pps2_lam0.4_temp1.0 on gpu1 pid=3767783 but output exists
[2026-05-05 12:45:13] launched h2_pps4_lam1.0_temp1.0 on gpu1 pid=3778927
[2026-05-05 12:45:28] failed h2_pps4_lam1.0_temp1.0 on gpu1 pid=3778927 exit=-1
[2026-05-05 12:45:28] launched h3_pps4_lam1.0_temp1.0 on gpu1 pid=3779683
[2026-05-05 12:45:43] failed h3_pps4_lam1.0_temp1.0 on gpu1 pid=3779683 exit=-1
[2026-05-05 12:45:43] launched top1_pps4_lam0.8_temp1.0 on gpu1 pid=3780155
[2026-05-05 12:46:08] nonzero exit (-1) for top1_pps2_lam0.2_temp1.0 on gpu5 pid=3769555 but output exists
[2026-05-05 12:46:08] launched top2_pps4_lam0.8_temp1.0 on gpu5 pid=3780510
[2026-05-05 12:46:13] nonzero exit (-1) for top2_pps2_lam0.2_temp1.0 on gpu4 pid=3770095 but output exists
[2026-05-05 12:46:13] launched top3_pps4_lam0.8_temp1.0 on gpu4 pid=3780656
[2026-05-05 12:46:23] nonzero exit (-1) for top3_pps2_lam0.4_temp1.0 on gpu2 pid=3768268 but output exists
[2026-05-05 12:46:23] launched h2_pps4_lam0.8_temp1.0 on gpu2 pid=3781366
[2026-05-05 12:46:38] failed h2_pps4_lam0.8_temp1.0 on gpu2 pid=3781366 exit=-1
[2026-05-05 12:46:38] launched h3_pps4_lam0.8_temp1.0 on gpu2 pid=3782494
[2026-05-05 12:46:53] failed h3_pps4_lam0.8_temp1.0 on gpu2 pid=3782494 exit=-1
[2026-05-05 12:46:53] launched top1_pps4_lam0.6_temp1.0 on gpu2 pid=3782840
[2026-05-05 12:47:13] nonzero exit (-1) for top2_pps4_lam1.0_temp1.0 on gpu7 pid=3773141 but output exists
[2026-05-05 12:47:13] launched top2_pps4_lam0.6_temp1.0 on gpu7 pid=3784336
[2026-05-05 12:47:23] nonzero exit (-1) for top3_pps2_lam0.2_temp1.0 on gpu0 pid=3771923 but output exists
[2026-05-05 12:47:23] launched top3_pps4_lam0.6_temp1.0 on gpu0 pid=3784682
[2026-05-05 12:47:23] finished top1_pps4_lam1.0_temp1.0 on gpu3 pid=3772953
[2026-05-05 12:47:23] launched h2_pps4_lam0.6_temp1.0 on gpu3 pid=3784690
[2026-05-05 12:47:38] failed h2_pps4_lam0.6_temp1.0 on gpu3 pid=3784690 exit=-1
[2026-05-05 12:47:38] launched h3_pps4_lam0.6_temp1.0 on gpu3 pid=3786074
[2026-05-05 12:47:53] failed h3_pps4_lam0.6_temp1.0 on gpu3 pid=3786074 exit=-1
[2026-05-05 12:47:53] launched top1_pps4_lam0.4_temp1.0 on gpu3 pid=3786420
[2026-05-05 12:49:48] nonzero exit (-1) for top3_pps4_lam1.0_temp1.0 on gpu6 pid=3778515 but output exists
[2026-05-05 12:49:48] launched top2_pps4_lam0.4_temp1.0 on gpu6 pid=3790306
[2026-05-05 12:50:18] nonzero exit (-1) for top1_pps4_lam0.8_temp1.0 on gpu1 pid=3780155 but output exists
[2026-05-05 12:50:18] launched top3_pps4_lam0.4_temp1.0 on gpu1 pid=3790924
[2026-05-05 12:50:28] nonzero exit (-1) for top2_pps4_lam0.8_temp1.0 on gpu5 pid=3780510 but output exists
[2026-05-05 12:50:28] launched h2_pps4_lam0.4_temp1.0 on gpu5 pid=3791267
[2026-05-05 12:50:43] nonzero exit (-1) for top3_pps4_lam0.8_temp1.0 on gpu4 pid=3780656 but output exists
[2026-05-05 12:50:43] launched h3_pps4_lam0.4_temp1.0 on gpu4 pid=3792314
[2026-05-05 12:50:43] failed h2_pps4_lam0.4_temp1.0 on gpu5 pid=3791267 exit=1
[2026-05-05 12:50:43] launched top1_pps4_lam0.2_temp1.0 on gpu5 pid=3792322
[2026-05-05 12:50:58] failed h3_pps4_lam0.4_temp1.0 on gpu4 pid=3792314 exit=-1
[2026-05-05 12:50:58] launched top2_pps4_lam0.2_temp1.0 on gpu4 pid=3793135
[2026-05-05 12:51:13] nonzero exit (-1) for top1_pps4_lam0.6_temp1.0 on gpu2 pid=3782840 but output exists
[2026-05-05 12:51:13] launched top3_pps4_lam0.2_temp1.0 on gpu2 pid=3793544
[2026-05-05 12:51:28] nonzero exit (-1) for top2_pps4_lam0.6_temp1.0 on gpu7 pid=3784336 but output exists
[2026-05-05 12:51:28] launched h2_pps4_lam0.2_temp1.0 on gpu7 pid=3793953
[2026-05-05 12:51:43] failed h2_pps4_lam0.2_temp1.0 on gpu7 pid=3793953 exit=-1
[2026-05-05 12:51:43] launched h3_pps4_lam0.2_temp1.0 on gpu7 pid=3795577
[2026-05-05 12:51:54] finished top3_pps4_lam0.6_temp1.0 on gpu0 pid=3784682
[2026-05-05 12:54:53] finished top3_pps4_lam0.4_temp1.0 on gpu1 pid=3790924
[2026-05-05 12:55:25] finished top3_pps4_lam0.2_temp1.0 on gpu2 pid=3793544
[2026-05-05 12:55:25] finished top1_pps4_lam0.4_temp1.0 on gpu3 pid=3786420
[2026-05-05 12:55:25] finished top2_pps4_lam0.2_temp1.0 on gpu4 pid=3793135
[2026-05-05 12:55:25] finished top1_pps4_lam0.2_temp1.0 on gpu5 pid=3792322
[2026-05-05 12:55:25] finished top2_pps4_lam0.4_temp1.0 on gpu6 pid=3790306
[2026-05-05 12:55:25] failed h3_pps4_lam0.2_temp1.0 on gpu7 pid=3795577 exit=1
[2026-05-05 14:18:26] starting temp=1.0 lambda grid on GPUs: 0,1,2,3,4,5,6,7
[2026-05-05 14:18:26] out_root=/home/sunhc/diffusion/sad/eval/mauve_temp10_lambda_grid_8gpu
[2026-05-05 14:18:26] num_samples=1024 sample_batch_size=256 mauve_batch_size=16
[2026-05-05 14:18:26] models=h2,h3
[2026-05-05 14:18:26] launched h2_pps1_lam1.0_temp1.0 on gpu0 pid=3837440
[2026-05-05 14:18:26] launched h3_pps1_lam1.0_temp1.0 on gpu1 pid=3837446
[2026-05-05 14:18:26] launched h2_pps1_lam0.8_temp1.0 on gpu2 pid=3837452
[2026-05-05 14:18:26] launched h3_pps1_lam0.8_temp1.0 on gpu3 pid=3837458
[2026-05-05 14:18:26] launched h2_pps1_lam0.6_temp1.0 on gpu4 pid=3837464
[2026-05-05 14:18:26] launched h3_pps1_lam0.6_temp1.0 on gpu5 pid=3837470
[2026-05-05 14:18:26] launched h2_pps1_lam0.4_temp1.0 on gpu6 pid=3837476
[2026-05-05 14:18:26] launched h3_pps1_lam0.4_temp1.0 on gpu7 pid=3837545
[2026-05-05 14:26:51] nonzero exit (-1) for h2_pps1_lam0.4_temp1.0 on gpu6 pid=3837476 but output exists
[2026-05-05 14:26:51] launched h2_pps1_lam0.2_temp1.0 on gpu6 pid=3851714
[2026-05-05 14:27:36] nonzero exit (-1) for h2_pps1_lam0.6_temp1.0 on gpu4 pid=3837464 but output exists
[2026-05-05 14:27:36] launched h3_pps1_lam0.2_temp1.0 on gpu4 pid=3852865
[2026-05-05 14:27:51] nonzero exit (-1) for h2_pps1_lam1.0_temp1.0 on gpu0 pid=3837440 but output exists
[2026-05-05 14:27:51] launched h2_pps2_lam1.0_temp1.0 on gpu0 pid=3853477
[2026-05-05 14:27:51] finished h2_pps1_lam0.8_temp1.0 on gpu2 pid=3837452
[2026-05-05 14:27:51] launched h3_pps2_lam1.0_temp1.0 on gpu2 pid=3853486
[2026-05-05 14:28:46] nonzero exit (-1) for h3_pps1_lam0.4_temp1.0 on gpu7 pid=3837545 but output exists
[2026-05-05 14:28:46] launched h2_pps2_lam0.8_temp1.0 on gpu7 pid=3854930
[2026-05-05 14:29:31] nonzero exit (-1) for h3_pps1_lam0.6_temp1.0 on gpu5 pid=3837470 but output exists
[2026-05-05 14:29:31] launched h3_pps2_lam0.8_temp1.0 on gpu5 pid=3855904
[2026-05-05 14:29:56] nonzero exit (-1) for h3_pps1_lam0.8_temp1.0 on gpu3 pid=3837458 but output exists
[2026-05-05 14:29:56] launched h2_pps2_lam0.6_temp1.0 on gpu3 pid=3856610
[2026-05-05 14:30:11] nonzero exit (-1) for h3_pps1_lam1.0_temp1.0 on gpu1 pid=3837446 but output exists
[2026-05-05 14:30:11] launched h3_pps2_lam0.6_temp1.0 on gpu1 pid=3857057
[2026-05-05 14:34:16] nonzero exit (-1) for h2_pps2_lam1.0_temp1.0 on gpu0 pid=3853477 but output exists
[2026-05-05 14:34:16] launched h2_pps2_lam0.4_temp1.0 on gpu0 pid=3864552
[2026-05-05 14:34:26] nonzero exit (-1) for h2_pps1_lam0.2_temp1.0 on gpu6 pid=3851714 but output exists
[2026-05-05 14:34:26] launched h3_pps2_lam0.4_temp1.0 on gpu6 pid=3865562
[2026-05-05 14:35:06] nonzero exit (-1) for h2_pps2_lam0.8_temp1.0 on gpu7 pid=3854930 but output exists
[2026-05-05 14:35:06] launched h2_pps2_lam0.2_temp1.0 on gpu7 pid=3866783
[2026-05-05 14:35:21] nonzero exit (-1) for h3_pps2_lam1.0_temp1.0 on gpu2 pid=3853486 but output exists
[2026-05-05 14:35:21] launched h3_pps2_lam0.2_temp1.0 on gpu2 pid=3867356
[2026-05-05 14:36:06] nonzero exit (-1) for h2_pps2_lam0.6_temp1.0 on gpu3 pid=3856610 but output exists
[2026-05-05 14:36:06] launched h2_pps4_lam1.0_temp1.0 on gpu3 pid=3868312
[2026-05-05 14:36:42] nonzero exit (-1) for h3_pps1_lam0.2_temp1.0 on gpu4 pid=3852865 but output exists
[2026-05-05 14:36:42] launched h3_pps4_lam1.0_temp1.0 on gpu4 pid=3869198
[2026-05-05 14:37:02] nonzero exit (-1) for h3_pps2_lam0.8_temp1.0 on gpu5 pid=3855904 but output exists
[2026-05-05 14:37:02] launched h2_pps4_lam0.8_temp1.0 on gpu5 pid=3869875
[2026-05-05 14:37:32] nonzero exit (-1) for h3_pps2_lam0.6_temp1.0 on gpu1 pid=3857057 but output exists
[2026-05-05 14:37:32] launched h3_pps4_lam0.8_temp1.0 on gpu1 pid=3871766
[2026-05-05 14:40:22] nonzero exit (-1) for h2_pps2_lam0.4_temp1.0 on gpu0 pid=3864552 but output exists
[2026-05-05 14:40:22] launched h2_pps4_lam0.6_temp1.0 on gpu0 pid=3878936
[2026-05-05 14:40:42] nonzero exit (-1) for h2_pps2_lam0.2_temp1.0 on gpu7 pid=3866783 but output exists
[2026-05-05 14:40:42] launched h3_pps4_lam0.6_temp1.0 on gpu7 pid=3879656
[2026-05-05 14:41:02] nonzero exit (-1) for h2_pps4_lam1.0_temp1.0 on gpu3 pid=3868312 but output exists
[2026-05-05 14:41:02] launched h2_pps4_lam0.4_temp1.0 on gpu3 pid=3880390
[2026-05-05 14:41:32] nonzero exit (-1) for h3_pps2_lam0.4_temp1.0 on gpu6 pid=3865562 but output exists
[2026-05-05 14:41:32] launched h3_pps4_lam0.4_temp1.0 on gpu6 pid=3881244
[2026-05-05 14:41:42] nonzero exit (-1) for h3_pps2_lam0.2_temp1.0 on gpu2 pid=3867356 but output exists
[2026-05-05 14:41:42] launched h2_pps4_lam0.2_temp1.0 on gpu2 pid=3881838
[2026-05-05 14:42:02] nonzero exit (-1) for h2_pps4_lam0.8_temp1.0 on gpu5 pid=3869875 but output exists
[2026-05-05 14:42:02] launched h3_pps4_lam0.2_temp1.0 on gpu5 pid=3882312
[2026-05-05 14:45:09] finished h2_pps4_lam0.6_temp1.0 on gpu0 pid=3878936
[2026-05-05 14:45:09] finished h3_pps4_lam0.8_temp1.0 on gpu1 pid=3871766
[2026-05-05 14:46:16] finished h2_pps4_lam0.2_temp1.0 on gpu2 pid=3881838
[2026-05-05 14:46:16] finished h2_pps4_lam0.4_temp1.0 on gpu3 pid=3880390
[2026-05-05 14:46:16] finished h3_pps4_lam1.0_temp1.0 on gpu4 pid=3869198
[2026-05-05 14:46:56] finished h3_pps4_lam0.2_temp1.0 on gpu5 pid=3882312
[2026-05-05 14:46:56] finished h3_pps4_lam0.4_temp1.0 on gpu6 pid=3881244
[2026-05-05 14:46:56] finished h3_pps4_lam0.6_temp1.0 on gpu7 pid=3879656
[2026-05-05 14:46:56] all temp=1.0 lambda grid jobs completed