boot log oozyy7xuhdydoh
Browse files- logs/boot-oozyy7xuhdydoh.log +1144 -0
logs/boot-oozyy7xuhdydoh.log
CHANGED
|
@@ -18387,3 +18387,1147 @@ worker-comfyui - Websocket receive timed out. Still waiting...
|
|
| 18387 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18388 |
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18389 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18390 |
15%|█▌ | 3/20 [04:07<23:21, 82.46s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18391 |
20%|██ | 4/20 [05:29<21:59, 82.44s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18392 |
25%|██▌ | 5/20 [06:52<20:35, 82.39s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18393 |
30%|███ | 6/20 [08:14<19:13, 82.36s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18387 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18388 |
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18389 |
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18390 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18391 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18392 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18393 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18394 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18395 |
+
[minimax] boot log shipped
|
| 18396 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18397 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18398 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18399 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18400 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18401 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18402 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18403 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18404 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18405 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18406 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18407 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18408 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18409 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18410 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18411 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18412 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18413 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18414 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18415 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18416 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18417 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18418 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18419 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18420 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18421 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18422 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18423 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18424 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18425 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18426 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18427 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18428 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18429 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18430 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18431 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18432 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18433 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18434 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18435 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18436 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18437 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18438 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18439 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18440 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18441 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18442 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18443 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18444 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18445 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18446 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18447 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18448 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18449 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18450 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18451 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18452 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18453 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18454 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18455 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18456 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18457 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18458 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18459 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18460 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18461 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18462 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18463 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18464 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18465 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18466 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18467 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18468 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18469 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18470 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18471 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18472 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18473 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18474 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18475 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18476 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18477 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18478 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18479 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18480 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18481 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18482 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18483 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18484 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18485 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18486 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18487 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18488 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18489 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18490 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18491 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18492 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18493 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18494 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18495 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18496 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18497 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18498 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18499 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18500 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18501 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18502 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18503 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18504 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18505 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18506 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18507 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18508 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18509 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18510 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18511 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18512 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18513 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18514 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18515 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18516 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18517 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=51976 MB
|
| 18518 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18519 |
+
|
| 18520 |
15%|█▌ | 3/20 [04:07<23:21, 82.46s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 18521 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798e8fce0770: Actual Resident VRAM = 4992 MB
|
| 18522 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bfeae3b30: Actual Resident VRAM = 608 MB
|
| 18523 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bff8bcd00: Actual Resident VRAM = 25888 MB
|
| 18524 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798d2e81acc0: Actual Resident VRAM = 20000 MB
|
| 18525 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 51488 MB
|
| 18526 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18527 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18528 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18529 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18530 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18531 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18532 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18533 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18534 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18535 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18536 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18537 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18538 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18539 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18540 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18541 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18542 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18543 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18544 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18545 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18546 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18547 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18548 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18549 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18550 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18551 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18552 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18553 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18554 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18555 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18556 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18557 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18558 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18559 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18560 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18561 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18562 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18563 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18564 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18565 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18566 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18567 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18568 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18569 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18570 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18571 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18572 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18573 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18574 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18575 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18576 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18577 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18578 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18579 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18580 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18581 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18582 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18583 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18584 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18585 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18586 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18587 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18588 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18589 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18590 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18591 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18592 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18593 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18594 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18595 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18596 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18597 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18598 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18599 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18600 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18601 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18602 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18603 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18604 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18605 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18606 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18607 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18608 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18609 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18610 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18611 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18612 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18613 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18614 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18615 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18616 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18617 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18618 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18619 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18620 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18621 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18622 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18623 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18624 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18625 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18626 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18627 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18628 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18629 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18630 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18631 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18632 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18633 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18634 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18635 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18636 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18637 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18638 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18639 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18640 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18641 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18642 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18643 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18644 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18645 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18646 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18647 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18648 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18649 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18650 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18651 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18652 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18653 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18654 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18655 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18656 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18657 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18658 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18659 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18660 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18661 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18662 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18663 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18664 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18665 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18666 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18667 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18668 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18669 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18670 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18671 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18672 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18673 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18674 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18675 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18676 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18677 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18678 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18679 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18680 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18681 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18682 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18683 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18684 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18685 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18686 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18687 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18688 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18689 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18690 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18691 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18692 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18693 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18694 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18695 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18696 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18697 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18698 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18699 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18700 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18701 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18702 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18703 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18704 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18705 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18706 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18707 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18708 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18709 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18710 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18711 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18712 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18713 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18714 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18715 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18716 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18717 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18718 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18719 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18720 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18721 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18722 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18723 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18724 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18725 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18726 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18727 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18728 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18729 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18730 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18731 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18732 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18733 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18734 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18735 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18736 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18737 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18738 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18739 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18740 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18741 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18742 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18743 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18744 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18745 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18746 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18747 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18748 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18749 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18750 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18751 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18752 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18753 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18754 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18755 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18756 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18757 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18758 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18759 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18760 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18761 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18762 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18763 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18764 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18765 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18766 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18767 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18768 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18769 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18770 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18771 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18772 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18773 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18774 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18775 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18776 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18777 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18778 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18779 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18780 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18781 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18782 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18783 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18784 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18785 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18786 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18787 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18788 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18789 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18790 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18791 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18792 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18793 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18794 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18795 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18796 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18797 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18798 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18799 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18800 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18801 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18802 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18803 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18804 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18805 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18806 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18807 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18808 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18809 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18810 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18811 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18812 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18813 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18814 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18815 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18816 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18817 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18818 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18819 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18820 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18821 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18822 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18823 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18824 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18825 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18826 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18827 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18828 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18829 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18830 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=51976 MB
|
| 18831 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18832 |
+
|
| 18833 |
20%|██ | 4/20 [05:29<21:59, 82.44s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 18834 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798e8fce0770: Actual Resident VRAM = 4992 MB
|
| 18835 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bfeae3b30: Actual Resident VRAM = 608 MB
|
| 18836 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bff8bcd00: Actual Resident VRAM = 25888 MB
|
| 18837 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798d2e81acc0: Actual Resident VRAM = 20000 MB
|
| 18838 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 51488 MB
|
| 18839 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18840 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18841 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18842 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18843 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18844 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18845 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18846 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18847 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18848 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18849 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18850 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18851 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18852 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18853 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18854 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18855 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18856 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18857 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18858 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18859 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18860 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18861 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18862 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18863 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18864 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18865 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18866 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18867 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18868 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18869 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18870 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18871 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18872 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18873 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18874 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18875 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18876 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18877 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18878 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18879 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18880 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18881 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18882 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18883 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18884 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18885 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18886 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18887 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18888 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18889 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18890 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18891 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18892 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18893 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18894 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18895 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18896 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18897 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18898 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18899 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18900 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18901 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18902 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18903 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18904 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18905 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18906 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18907 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18908 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18909 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18910 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18911 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18912 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18913 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18914 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18915 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18916 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18917 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18918 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18919 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18920 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18921 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18922 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18923 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18924 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18925 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18926 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18927 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18928 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18929 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18930 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18931 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18932 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18933 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18934 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18935 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18936 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18937 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18938 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18939 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18940 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18941 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18942 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18943 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18944 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18945 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18946 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18947 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18948 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18949 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18950 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18951 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18952 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18953 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18954 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18955 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18956 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18957 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18958 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18959 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18960 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18961 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18962 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18963 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18964 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18965 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18966 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18967 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18968 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18969 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18970 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18971 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18972 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18973 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18974 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18975 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18976 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18977 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18978 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18979 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18980 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18981 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18982 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18983 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18984 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18985 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 18986 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18987 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 18988 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18989 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18990 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18991 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18992 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18993 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18994 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18995 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18996 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 18997 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 18998 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 18999 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19000 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19001 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19002 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19003 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19004 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19005 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19006 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19007 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19008 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19009 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19010 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19011 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19012 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19013 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19014 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19015 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19016 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19017 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19018 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19019 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19020 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19021 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19022 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19023 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19024 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19025 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19026 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19027 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19028 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19029 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19030 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19031 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19032 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19033 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19034 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19035 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19036 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19037 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19038 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19039 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19040 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19041 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19042 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19043 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19044 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19045 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19046 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19047 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19048 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19049 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19050 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19051 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19052 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19053 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19054 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19055 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19056 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19057 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19058 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19059 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19060 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19061 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19062 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19063 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19064 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19065 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19066 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19067 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19068 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19069 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19070 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19071 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19072 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19073 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19074 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19075 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19076 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19077 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19078 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19079 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19080 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19081 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19082 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19083 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19084 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19085 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19086 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19087 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19088 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19089 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19090 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19091 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19092 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19093 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19094 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19095 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19096 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19097 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19098 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19099 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19100 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19101 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19102 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19103 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19104 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19105 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19106 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19107 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19108 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19109 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19110 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19111 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19112 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19113 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19114 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19115 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19116 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19117 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19118 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19119 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19120 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19121 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19122 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19123 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19124 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19125 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19126 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19127 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19128 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19129 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19130 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19131 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19132 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19133 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19134 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19135 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19136 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19137 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19138 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19139 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19140 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19141 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19142 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19143 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=51976 MB
|
| 19144 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19145 |
+
|
| 19146 |
25%|██▌ | 5/20 [06:52<20:35, 82.39s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 19147 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798e8fce0770: Actual Resident VRAM = 4992 MB
|
| 19148 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bfeae3b30: Actual Resident VRAM = 608 MB
|
| 19149 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bff8bcd00: Actual Resident VRAM = 25888 MB
|
| 19150 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798d2e81acc0: Actual Resident VRAM = 20000 MB
|
| 19151 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 51488 MB
|
| 19152 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19153 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19154 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19155 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19156 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19157 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19158 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19159 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19160 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19161 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19162 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19163 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19164 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19165 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19166 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19167 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19168 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19169 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19170 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19171 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19172 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19173 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19174 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19175 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19176 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19177 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19178 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19179 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19180 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19181 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19182 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19183 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19184 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19185 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19186 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19187 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19188 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19189 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19190 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19191 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19192 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19193 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19194 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19195 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19196 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19197 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19198 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19199 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19200 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19201 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19202 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19203 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19204 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19205 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19206 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19207 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19208 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19209 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19210 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19211 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19212 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19213 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19214 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19215 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19216 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19217 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19218 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19219 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19220 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19221 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19222 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19223 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19224 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19225 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19226 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19227 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19228 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19229 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19230 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19231 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19232 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19233 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19234 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19235 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19236 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19237 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19238 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19239 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19240 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19241 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19242 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19243 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19244 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19245 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19246 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19247 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19248 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19249 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19250 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19251 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19252 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19253 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19254 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19255 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19256 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19257 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19258 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19259 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19260 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19261 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19262 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19263 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19264 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19265 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19266 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19267 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19268 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19269 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19270 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19271 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19272 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19273 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19274 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19275 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19276 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19277 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19278 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19279 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19280 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19281 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19282 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19283 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19284 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19285 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19286 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19287 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19288 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19289 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19290 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19291 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19292 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19293 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19294 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19295 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19296 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19297 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19298 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19299 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19300 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19301 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19302 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19303 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19304 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19305 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19306 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19307 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19308 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19309 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19310 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19311 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19312 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19313 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19314 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19315 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19316 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19317 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19318 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19319 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19320 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19321 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19322 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19323 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19324 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19325 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19326 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19327 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19328 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19329 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19330 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19331 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19332 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19333 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19334 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19335 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19336 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19337 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19338 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19339 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19340 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19341 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19342 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19343 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19344 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19345 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19346 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19347 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19348 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19349 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19350 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19351 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19352 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19353 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19354 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19355 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19356 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19357 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19358 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19359 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19360 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19361 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19362 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19363 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19364 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19365 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19366 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19367 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19368 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19369 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19370 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19371 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19372 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19373 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19374 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19375 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19376 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19377 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19378 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19379 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19380 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19381 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19382 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19383 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19384 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19385 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19386 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19387 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19388 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19389 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19390 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19391 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19392 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19393 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19394 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19395 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19396 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19397 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19398 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19399 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19400 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19401 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19402 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19403 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19404 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19405 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19406 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19407 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19408 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19409 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19410 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19411 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19412 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19413 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19414 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19415 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19416 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19417 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19418 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19419 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19420 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19421 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19422 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19423 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19424 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19425 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19426 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19427 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19428 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19429 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19430 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19431 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19432 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19433 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19434 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19435 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19436 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19437 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19438 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19439 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19440 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19441 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19442 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19443 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19444 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19445 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19446 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19447 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19448 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19449 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19450 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19451 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19452 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19453 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19454 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19455 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19456 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=51976 MB
|
| 19457 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19458 |
+
|
| 19459 |
30%|███ | 6/20 [08:14<19:13, 82.36s/it][36m[DEBUG][0m aimdo: /project/src/model-vbar.c:57:DEBUG:---------------- VBAR Usage ---------------
|
| 19460 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798e8fce0770: Actual Resident VRAM = 4992 MB
|
| 19461 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bfeae3b30: Actual Resident VRAM = 608 MB
|
| 19462 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x797bff8bcd00: Actual Resident VRAM = 25888 MB
|
| 19463 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:86:DEBUG:VBAR 0x798d2e81acc0: Actual Resident VRAM = 20000 MB
|
| 19464 |
+
[36m[DEBUG][0m aimdo: /project/src/model-vbar.c:90:DEBUG:Total VRAM for VBARs: 51488 MB
|
| 19465 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19466 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19467 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19468 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19469 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19470 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19471 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19472 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19473 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19474 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19475 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19476 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19477 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19478 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19479 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19480 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19481 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19482 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19483 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19484 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19485 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19486 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19487 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19488 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19489 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19490 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19491 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19492 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19493 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19494 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19495 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19496 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19497 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19498 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19499 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19500 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|
| 19501 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19502 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19503 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19504 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19505 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19506 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19507 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19508 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19509 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19510 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19511 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19512 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19513 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19514 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19515 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19516 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19517 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19518 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19519 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19520 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19521 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19522 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19523 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19524 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19525 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:170:DEBUG:cuda_budget_deficit: device memory poll free=25920 MB total=97249 MB deficit_sync=-25664 MB recorded=57925 MB
|
| 19526 |
+
[36m[DEBUG][0m aimdo: /project/src/control.c:174:DEBUG:cuda_budget_deficit: prevailing method cuMemGetInfo
|
| 19527 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19528 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19529 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19530 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19531 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19532 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19533 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19534 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19535 |
+
[36m[DEBUG][0m Backend cuda selected for rms_rope_split_half_
|
| 19536 |
+
[36m[DEBUG][0m Backend cuda selected for int8_linear
|
| 19537 |
+
worker-comfyui - Websocket receive timed out. Still waiting...
|