exp017_anima_multiband/mb3_s0: ship-on-completion from Colab

exp017_anima_multiband/mb3_s0/aleph_relay.canonical.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:22eabc70bb1e4499181bafd12ae1cb618130e94b7120ad9b770bc8fb559d9ce3
+size 11380544

exp017_anima_multiband/mb3_s0/aleph_relay.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf7ad9c4636acb4508cf2bcd224759f60075826667003587c7db2d82e7be1126
+size 11445063

exp017_anima_multiband/mb3_s0/aleph_relay.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e973620286425da059c7e034412167845323b530a9afb8437f3bfe5a928e004
+size 11387312

exp017_anima_multiband/mb3_s0/e017_mb3_s0.toml

@@ -0,0 +1,48 @@
+# e017_mb3_s0 — generated by anima_closeout.ipynb
+output_dir = '/content/anima_campaign/runs/e017_mb3_s0'
+dataset = '/content/anima_campaign/tomls/dataset_train.toml'
+seed = 0
+bf16_master_weights = true   # the exp004 gate-freeze fix
+epochs = 1000
+max_steps = 10000
+micro_batch_size_per_gpu = 8
+pipeline_stages = 1
+gradient_accumulation_steps = 1
+gradient_clipping = 0
+save_every_n_steps = 2500
+checkpoint_every_n_minutes = 30
+activation_checkpointing = false   # 96GB rider
+save_dtype = 'bfloat16'
+caching_batch_size = 8
+eval_datasets = [{name = 'heldout', config = '/content/anima_campaign/tomls/dataset_eval.toml'}]
+eval_every_n_steps = 1000
+eval_before_first_step = true
+eval_micro_batch_size_per_gpu = 8
+eval_gradient_accumulation_steps = 1
+
+[model]
+type = 'anima'
+transformer_path = '/content/anima_campaign/models/anima/split_files/diffusion_models/anima-base-v1.0.safetensors'
+llm_path = '/content/anima_campaign/models/anima/split_files/text_encoders/qwen_3_06b_base.safetensors'
+vae_path = '/content/anima_campaign/models/anima/split_files/vae/qwen_image_vae.safetensors'
+dtype = 'bfloat16'
+timestep_sample_method = 'logit_normal'
+aleph_relay = true
+aleph_relay_mode = 'multiband3'
+aleph_relay_rank = 16
+aleph_relay_every = 1
+aleph_relay_lr = 1e-3
+self_attn_lr = 0
+cross_attn_lr = 0
+mlp_lr = 0
+mod_lr = 0
+llm_adapter_lr = 0   # NON-NEGOTIABLE (anima-trainer law)
+
+[optimizer]
+type = 'adam'   # pure Adam — never adamw on aleph paths
+lr = 0
+betas = [0.9, 0.99]
+weight_decay = 0.0
+
+[monitoring]
+enable_wandb = false

exp017_anima_multiband/mb3_s0/eval_curve.json

@@ -0,0 +1,462 @@
+{
+ "heldout/loss_quantile_0.10": [
+  [
+   0,
+   0.16599799692630768
+  ],
+  [
+   1000,
+   0.16337965428829193
+  ],
+  [
+   2000,
+   0.1639113575220108
+  ],
+  [
+   3000,
+   0.1616498976945877
+  ],
+  [
+   4000,
+   0.16123060882091522
+  ],
+  [
+   5000,
+   0.16022039949893951
+  ],
+  [
+   6000,
+   0.15988416969776154
+  ],
+  [
+   7000,
+   0.15969954431056976
+  ],
+  [
+   8000,
+   0.15951265394687653
+  ],
+  [
+   9000,
+   0.15950831770896912
+  ],
+  [
+   10000,
+   0.15953323245048523
+  ]
+ ],
+ "heldout/loss_quantile_0.20": [
+  [
+   0,
+   0.12855228781700134
+  ],
+  [
+   1000,
+   0.12580077350139618
+  ],
+  [
+   2000,
+   0.1261817365884781
+  ],
+  [
+   3000,
+   0.12414102256298065
+  ],
+  [
+   4000,
+   0.12379635125398636
+  ],
+  [
+   5000,
+   0.12287270277738571
+  ],
+  [
+   6000,
+   0.12250229716300964
+  ],
+  [
+   7000,
+   0.12232135981321335
+  ],
+  [
+   8000,
+   0.12209752947092056
+  ],
+  [
+   9000,
+   0.12209073454141617
+  ],
+  [
+   10000,
+   0.12202215939760208
+  ]
+ ],
+ "heldout/loss_quantile_0.30": [
+  [
+   0,
+   0.11035258322954178
+  ],
+  [
+   1000,
+   0.10706117004156113
+  ],
+  [
+   2000,
+   0.10676281899213791
+  ],
+  [
+   3000,
+   0.1055157482624054
+  ],
+  [
+   4000,
+   0.10489069670438766
+  ],
+  [
+   5000,
+   0.10479593276977539
+  ],
+  [
+   6000,
+   0.10413006693124771
+  ],
+  [
+   7000,
+   0.1039654016494751
+  ],
+  [
+   8000,
+   0.10378488153219223
+  ],
+  [
+   9000,
+   0.10357914865016937
+  ],
+  [
+   10000,
+   0.1033618301153183
+  ]
+ ],
+ "heldout/loss_quantile_0.40": [
+  [
+   0,
+   0.09996877610683441
+  ],
+  [
+   1000,
+   0.09752071648836136
+  ],
+  [
+   2000,
+   0.09724651277065277
+  ],
+  [
+   3000,
+   0.09558578580617905
+  ],
+  [
+   4000,
+   0.09483996778726578
+  ],
+  [
+   5000,
+   0.09493996948003769
+  ],
+  [
+   6000,
+   0.09409952908754349
+  ],
+  [
+   7000,
+   0.09388314932584763
+  ],
+  [
+   8000,
+   0.0936349481344223
+  ],
+  [
+   9000,
+   0.09340900927782059
+  ],
+  [
+   10000,
+   0.09314996004104614
+  ]
+ ],
+ "heldout/loss_quantile_0.50": [
+  [
+   0,
+   0.09322996437549591
+  ],
+  [
+   1000,
+   0.09016970545053482
+  ],
+  [
+   2000,
+   0.08969684690237045
+  ],
+  [
+   3000,
+   0.08818637579679489
+  ],
+  [
+   4000,
+   0.08743215352296829
+  ],
+  [
+   5000,
+   0.08745754510164261
+  ],
+  [
+   6000,
+   0.08658429980278015
+  ],
+  [
+   7000,
+   0.08639784902334213
+  ],
+  [
+   8000,
+   0.08610570430755615
+  ],
+  [
+   9000,
+   0.08584776520729065
+  ],
+  [
+   10000,
+   0.08556636422872543
+  ]
+ ],
+ "heldout/loss_quantile_0.60": [
+  [
+   0,
+   0.08950993418693542
+  ],
+  [
+   1000,
+   0.08567140251398087
+  ],
+  [
+   2000,
+   0.0850827544927597
+  ],
+  [
+   3000,
+   0.08367945998907089
+  ],
+  [
+   4000,
+   0.08291463553905487
+  ],
+  [
+   5000,
+   0.08284605294466019
+  ],
+  [
+   6000,
+   0.08207394927740097
+  ],
+  [
+   7000,
+   0.08187572658061981
+  ],
+  [
+   8000,
+   0.08152744174003601
+  ],
+  [
+   9000,
+   0.08125205338001251
+  ],
+  [
+   10000,
+   0.0809599757194519
+  ]
+ ],
+ "heldout/loss_quantile_0.70": [
+  [
+   0,
+   0.08822756260633469
+  ],
+  [
+   1000,
+   0.08352743834257126
+  ],
+  [
+   2000,
+   0.08283019065856934
+  ],
+  [
+   3000,
+   0.08136025071144104
+  ],
+  [
+   4000,
+   0.08055517822504044
+  ],
+  [
+   5000,
+   0.08055227994918823
+  ],
+  [
+   6000,
+   0.07977219671010971
+  ],
+  [
+   7000,
+   0.07941807061433792
+  ],
+  [
+   8000,
+   0.07915136963129044
+  ],
+  [
+   9000,
+   0.07885309308767319
+  ],
+  [
+   10000,
+   0.07862167060375214
+  ]
+ ],
+ "heldout/loss_quantile_0.80": [
+  [
+   0,
+   0.08957124501466751
+  ],
+  [
+   1000,
+   0.08371134102344513
+  ],
+  [
+   2000,
+   0.08303967118263245
+  ],
+  [
+   3000,
+   0.0816778913140297
+  ],
+  [
+   4000,
+   0.08068321645259857
+  ],
+  [
+   5000,
+   0.08068323135375977
+  ],
+  [
+   6000,
+   0.08006388694047928
+  ],
+  [
+   7000,
+   0.0795588344335556
+  ],
+  [
+   8000,
+   0.07932315021753311
+  ],
+  [
+   9000,
+   0.07907573133707047
+  ],
+  [
+   10000,
+   0.0787273719906807
+  ]
+ ],
+ "heldout/loss_quantile_0.90": [
+  [
+   0,
+   0.09744053333997726
+  ],
+  [
+   1000,
+   0.0950915738940239
+  ],
+  [
+   2000,
+   0.09495202451944351
+  ],
+  [
+   3000,
+   0.09362520277500153
+  ],
+  [
+   4000,
+   0.09340228885412216
+  ],
+  [
+   5000,
+   0.09152020514011383
+  ],
+  [
+   6000,
+   0.09065908938646317
+  ],
+  [
+   7000,
+   0.08976077288389206
+  ],
+  [
+   8000,
+   0.08988749235868454
+  ],
+  [
+   9000,
+   0.0891101285815239
+  ],
+  [
+   10000,
+   0.08910766988992691
+  ]
+ ],
+ "heldout/loss": [
+  [
+   0,
+   0.10698343068361282
+  ],
+  [
+   1000,
+   0.10354819893836975
+  ],
+  [
+   2000,
+   0.10330043733119965
+  ],
+  [
+   3000,
+   0.10171351581811905
+  ],
+  [
+   4000,
+   0.10108278691768646
+  ],
+  [
+   5000,
+   0.10065425932407379
+  ],
+  [
+   6000,
+   0.0999743863940239
+  ],
+  [
+   7000,
+   0.09965340793132782
+  ],
+  [
+   8000,
+   0.09944724291563034
+  ],
+  [
+   9000,
+   0.09919177740812302
+  ],
+  [
+   10000,
+   0.09900557994842529
+  ]
+ ]
+}

exp017_anima_multiband/mb3_s0/log_tail.txt

@@ -0,0 +1,400 @@
+ 31%|███       | 175/567 [00:33<01:15,  5.19it/s]
+ 31%|███       | 176/567 [00:33<01:15,  5.20it/s]
+ 31%|███       | 177/567 [00:34<01:15,  5.18it/s]
+ 31%|███▏      | 178/567 [00:34<01:15,  5.19it/s]
+ 32%|███▏      | 179/567 [00:34<01:14,  5.18it/s]
+ 32%|███▏      | 180/567 [00:34<01:15,  5.16it/s]
+ 32%|███▏      | 181/567 [00:34<01:15,  5.14it/s]
+ 32%|███▏      | 182/567 [00:35<01:15,  5.13it/s]
+ 32%|███▏      | 183/567 [00:35<01:14,  5.15it/s]
+ 32%|███▏      | 184/567 [00:35<01:14,  5.16it/s]
+ 33%|███▎      | 185/567 [00:35<01:14,  5.14it/s]
+ 33%|███▎      | 186/567 [00:35<01:13,  5.16it/s]
+ 33%|███▎      | 187/567 [00:36<01:13,  5.16it/s]
+ 33%|███▎      | 188/567 [00:36<01:13,  5.15it/s]
+ 33%|███▎      | 189/567 [00:36<01:12,  5.24it/s]
+ 34%|███▎      | 190/567 [00:36<01:13,  5.15it/s]
+ 34%|███▎      | 191/567 [00:36<01:13,  5.14it/s]
+ 34%|███▍      | 192/567 [00:37<01:12,  5.15it/s]
+ 34%|███▍      | 193/567 [00:37<01:12,  5.16it/s]
+ 34%|███▍      | 194/567 [00:37<01:12,  5.15it/s]
+ 34%|███▍      | 195/567 [00:37<01:12,  5.16it/s]
+ 35%|███▍      | 196/567 [00:37<01:12,  5.14it/s]
+ 35%|███▍      | 197/567 [00:37<01:11,  5.15it/s]
+ 35%|███▍      | 198/567 [00:38<01:11,  5.16it/s]
+ 35%|███▌      | 199/567 [00:38<01:11,  5.14it/s]
+ 35%|███▌      | 200/567 [00:38<01:11,  5.16it/s]
+ 35%|███▌      | 201/567 [00:38<01:11,  5.14it/s]
+ 36%|███▌      | 202/567 [00:38<01:11,  5.13it/s]
+ 36%|███▌      | 203/567 [00:39<01:11,  5.12it/s]
+ 36%|███▌      | 204/567 [00:39<01:10,  5.14it/s]
+ 36%|███▌      | 205/567 [00:39<01:10,  5.15it/s]
+ 36%|███▋      | 206/567 [00:39<01:09,  5.16it/s]
+ 37%|███▋      | 207/567 [00:39<01:09,  5.17it/s]
+ 37%|███▋      | 208/567 [00:40<01:09,  5.18it/s]
+ 37%|███▋      | 209/567 [00:40<01:09,  5.18it/s]
+ 37%|███▋      | 210/567 [00:40<01:09,  5.16it/s]
+ 37%|███▋      | 211/567 [00:40<01:09,  5.13it/s]
+ 37%|███▋      | 212/567 [00:40<01:08,  5.15it/s]
+ 38%|███▊      | 213/567 [00:41<01:08,  5.16it/s]
+ 38%|███▊      | 214/567 [00:41<01:08,  5.17it/s]
+ 38%|███▊      | 215/567 [00:41<01:07,  5.18it/s]
+ 38%|███▊      | 216/567 [00:41<01:07,  5.18it/s]
+ 38%|███▊      | 217/567 [00:41<01:07,  5.15it/s]
+ 38%|███▊      | 218/567 [00:42<01:07,  5.16it/s]
+ 39%|███▊      | 219/567 [00:42<01:07,  5.17it/s]
+ 39%|███▉      | 220/567 [00:42<01:07,  5.17it/s]
+ 39%|███▉      | 221/567 [00:42<01:07,  5.15it/s]
+ 39%|███▉      | 222/567 [00:42<01:06,  5.16it/s]
+ 39%|███▉      | 223/567 [00:43<01:06,  5.17it/s]
+ 40%|███▉      | 224/567 [00:43<01:06,  5.18it/s]
+ 40%|███▉      | 225/567 [00:43<01:06,  5.15it/s]
+ 40%|███▉      | 226/567 [00:43<01:06,  5.16it/s]
+ 40%|████      | 227/567 [00:43<01:05,  5.17it/s]
+ 40%|████      | 228/567 [00:43<01:05,  5.15it/s]
+ 40%|████      | 229/567 [00:44<01:05,  5.14it/s]
+ 41%|████      | 230/567 [00:44<01:05,  5.15it/s]
+ 41%|████      | 231/567 [00:44<01:05,  5.13it/s]
+ 41%|████      | 232/567 [00:44<01:04,  5.16it/s]
+ 41%|████      | 233/567 [00:44<01:04,  5.15it/s]
+ 41%|████▏     | 234/567 [00:45<01:04,  5.17it/s]
+ 41%|████▏     | 235/567 [00:45<01:04,  5.19it/s]
+ 42%|████▏     | 236/567 [00:45<01:03,  5.20it/s]
+ 42%|████▏     | 237/567 [00:45<01:03,  5.21it/s]
+ 42%|████▏     | 238/567 [00:45<01:03,  5.19it/s]
+ 42%|████▏     | 239/567 [00:46<01:03,  5.20it/s]
+ 42%|████▏     | 240/567 [00:46<01:03,  5.18it/s]
+ 43%|████▎     | 241/567 [00:46<01:02,  5.20it/s]
+ 43%|████▎     | 242/567 [00:46<01:02,  5.21it/s]
+ 43%|████▎     | 243/567 [00:46<01:02,  5.18it/s]
+ 43%|████▎     | 244/567 [00:47<01:02,  5.17it/s]
+ 43%|████▎     | 245/567 [00:47<01:02,  5.16it/s]
+ 43%|████▎     | 246/567 [00:47<01:02,  5.18it/s]
+ 44%|████▎     | 247/567 [00:47<01:01,  5.19it/s]
+ 44%|████▎     | 248/567 [00:47<01:01,  5.17it/s]
+ 44%|████▍     | 249/567 [00:48<01:01,  5.19it/s]
+ 44%|████▍     | 250/567 [00:48<01:00,  5.20it/s]
+ 44%|████▍     | 251/567 [00:48<01:00,  5.18it/s]
+ 44%|████▍     | 252/567 [00:48<00:59,  5.26it/s]
+ 45%|████▍     | 253/567 [00:48<01:00,  5.19it/s]
+ 45%|████▍     | 254/567 [00:49<01:00,  5.17it/s]
+ 45%|████▍     | 255/567 [00:49<01:00,  5.19it/s]
+ 45%|████▌     | 256/567 [00:49<00:59,  5.19it/s]
+ 45%|████▌     | 257/567 [00:49<00:59,  5.18it/s]
+ 46%|████▌     | 258/567 [00:49<00:59,  5.19it/s]
+ 46%|████▌     | 259/567 [00:49<00:59,  5.17it/s]
+ 46%|████▌     | 260/567 [00:50<00:59,  5.19it/s]
+ 46%|████▌     | 261/567 [00:50<00:58,  5.20it/s]
+ 46%|████▌     | 262/567 [00:50<00:58,  5.18it/s]
+ 46%|████▋     | 263/567 [00:50<00:58,  5.19it/s]
+ 47%|████▋     | 264/567 [00:50<00:58,  5.17it/s]
+ 47%|████▋     | 265/567 [00:51<00:58,  5.16it/s]
+ 47%|████▋     | 266/567 [00:51<00:58,  5.15it/s]
+ 47%|████▋     | 267/567 [00:51<00:57,  5.17it/s]
+ 47%|████▋     | 268/567 [00:51<00:57,  5.19it/s]
+ 47%|████▋     | 269/567 [00:51<00:57,  5.20it/s]
+ 48%|████▊     | 270/567 [00:52<00:57,  5.21it/s]
+ 48%|████▊     | 271/567 [00:52<00:56,  5.21it/s]
+ 48%|████▊     | 272/567 [00:52<00:56,  5.22it/s]
+ 48%|████▊     | 273/567 [00:52<00:56,  5.19it/s]
+ 48%|████▊     | 274/567 [00:52<00:56,  5.17it/s]
+ 49%|████▊     | 275/567 [00:53<00:56,  5.19it/s]
+ 49%|████▊     | 276/567 [00:53<00:55,  5.20it/s]
+ 49%|████▉     | 277/567 [00:53<00:55,  5.21it/s]
+ 49%|████▉     | 278/567 [00:53<00:55,  5.21it/s]
+ 49%|████▉     | 279/567 [00:53<00:55,  5.21it/s]
+ 49%|████▉     | 280/567 [00:54<00:55,  5.19it/s]
+ 50%|████▉     | 281/567 [00:54<00:55,  5.19it/s]
+ 50%|████▉     | 282/567 [00:54<00:54,  5.20it/s]
+ 50%|████▉     | 283/567 [00:54<00:54,  5.21it/s]
+ 50%|█████     | 284/567 [00:54<00:54,  5.19it/s]
+ 50%|█████     | 285/567 [00:54<00:54,  5.20it/s]
+ 50%|█████     | 286/567 [00:55<00:53,  5.21it/s]
+ 51%|█████     | 287/567 [00:55<00:53,  5.22it/s]
+ 51%|█████     | 288/567 [00:55<00:53,  5.19it/s]
+ 51%|█████     | 289/567 [00:55<00:53,  5.20it/s]
+ 51%|█████     | 290/567 [00:55<00:53,  5.20it/s]
+ 51%|█████▏    | 291/567 [00:56<00:53,  5.17it/s]
+ 51%|█████▏    | 292/567 [00:56<00:53,  5.16it/s]
+ 52%|█████▏    | 293/567 [00:56<00:52,  5.17it/s]
+ 52%|█████▏    | 294/567 [00:56<00:52,  5.16it/s]
+ 52%|█████▏    | 295/567 [00:56<00:52,  5.17it/s]
+ 52%|█████▏    | 296/567 [00:57<00:52,  5.15it/s]
+ 52%|█████▏    | 297/567 [00:57<00:52,  5.17it/s]
+ 53%|█████▎    | 298/567 [00:57<00:52,  5.17it/s]
+ 53%|█████▎    | 299/567 [00:57<00:51,  5.19it/s]
+ 53%|█████▎    | 300/567 [00:57<00:51,  5.19it/s]
+ 53%|█████▎    | 301/567 [00:58<00:51,  5.17it/s]
+ 53%|█████▎    | 302/567 [00:58<00:51,  5.19it/s]
+ 53%|█████▎    | 303/567 [00:58<00:51,  5.17it/s]
+ 54%|█████▎    | 304/567 [00:58<00:50,  5.17it/s]
+ 54%|█████▍    | 305/567 [00:58<00:50,  5.17it/s]
+ 54%|█████▍    | 306/567 [00:59<00:50,  5.15it/s]
+ 54%|█████▍    | 307/567 [00:59<00:50,  5.13it/s]
+ 54%|█████▍    | 308/567 [00:59<00:50,  5.12it/s]
+ 54%|█████▍    | 309/567 [00:59<00:50,  5.14it/s]
+ 55%|█████▍    | 310/567 [00:59<00:49,  5.15it/s]
+ 55%|█████▍    | 311/567 [01:00<00:49,  5.13it/s]
+ 55%|█████▌    | 312/567 [01:00<00:49,  5.14it/s]
+ 55%|█████▌    | 313/567 [01:00<00:49,  5.16it/s]
+ 55%|█████▌    | 314/567 [01:00<00:49,  5.14it/s]
+ 56%|█████▌    | 315/567 [01:00<00:48,  5.23it/s]
+ 56%|█████▌    | 316/567 [01:00<00:48,  5.14it/s]
+ 56%|█████▌    | 317/567 [01:01<00:48,  5.13it/s]
+ 56%|█████▌    | 318/567 [01:01<00:48,  5.15it/s]
+ 56%|█████▋    | 319/567 [01:01<00:48,  5.16it/s]
+ 56%|█████▋    | 320/567 [01:01<00:48,  5.14it/s]
+ 57%|█████▋    | 321/567 [01:01<00:47,  5.16it/s]
+ 57%|█████▋    | 322/567 [01:02<00:47,  5.14it/s]
+ 57%|█████▋    | 323/567 [01:02<00:47,  5.15it/s]
+ 57%|█████▋    | 324/567 [01:02<00:47,  5.16it/s]
+ 57%|█████▋    | 325/567 [01:02<00:47,  5.15it/s]
+ 57%|█████▋    | 326/567 [01:02<00:46,  5.16it/s]
+ 58%|█████▊    | 327/567 [01:03<00:46,  5.14it/s]
+ 58%|█████▊    | 328/567 [01:03<00:46,  5.12it/s]
+ 58%|█████▊    | 329/567 [01:03<00:46,  5.12it/s]
+ 58%|█████▊    | 330/567 [01:03<00:46,  5.14it/s]
+ 58%|█████▊    | 331/567 [01:03<00:45,  5.15it/s]
+ 59%|█████▊    | 332/567 [01:04<00:45,  5.16it/s]
+ 59%|█████▊    | 333/567 [01:04<00:45,  5.17it/s]
+ 59%|█████▉    | 334/567 [01:04<00:45,  5.16it/s]
+ 59%|█████▉    | 335/567 [01:04<00:44,  5.17it/s]
+ 59%|█████▉    | 336/567 [01:04<00:44,  5.15it/s]
+ 59%|█████▉    | 337/567 [01:05<00:44,  5.14it/s]
+ 60%|█████▉    | 338/567 [01:05<00:44,  5.14it/s]
+ 60%|█████▉    | 339/567 [01:05<00:44,  5.16it/s]
+ 60%|█████▉    | 340/567 [01:05<00:43,  5.17it/s]
+ 60%|██████    | 341/567 [01:05<00:43,  5.17it/s]
+ 60%|██████    | 342/567 [01:06<00:43,  5.18it/s]
+ 60%|██████    | 343/567 [01:06<00:43,  5.15it/s]
+ 61%|██████    | 344/567 [01:06<00:43,  5.16it/s]
+ 61%|██████    | 345/567 [01:06<00:42,  5.17it/s]
+ 61%|██████    | 346/567 [01:06<00:42,  5.18it/s]
+ 61%|██████    | 347/567 [01:06<00:42,  5.15it/s]
+ 61%|██████▏   | 348/567 [01:07<00:42,  5.16it/s]
+ 62%|██████▏   | 349/567 [01:07<00:42,  5.17it/s]
+ 62%|██████▏   | 350/567 [01:07<00:41,  5.18it/s]
+ 62%|██████▏   | 351/567 [01:07<00:41,  5.16it/s]
+ 62%|██████▏   | 352/567 [01:07<00:41,  5.17it/s]
+ 62%|██████▏   | 353/567 [01:08<00:41,  5.17it/s]
+ 62%|██████▏   | 354/567 [01:08<00:41,  5.15it/s]
+ 63%|██████▎   | 355/567 [01:08<00:41,  5.13it/s]
+ 63%|██████▎   | 356/567 [01:08<00:40,  5.15it/s]
+ 63%|██████▎   | 357/567 [01:08<00:40,  5.13it/s]
+ 63%|██████▎   | 358/567 [01:09<00:40,  5.15it/s]
+ 63%|██████▎   | 359/567 [01:09<00:40,  5.13it/s]
+ 63%|██████▎   | 360/567 [01:09<00:40,  5.15it/s]
+ 64%|██████▎   | 361/567 [01:09<00:39,  5.16it/s]
+ 64%|██████▍   | 362/567 [01:09<00:39,  5.17it/s]
+ 64%|██████▍   | 363/567 [01:10<00:39,  5.17it/s]
+ 64%|██████▍   | 364/567 [01:10<00:39,  5.15it/s]
+ 64%|██████▍   | 365/567 [01:10<00:39,  5.16it/s]
+ 65%|██████▍   | 366/567 [01:10<00:39,  5.14it/s]
+ 65%|██████▍   | 367/567 [01:10<00:38,  5.16it/s]
+ 65%|██████▍   | 368/567 [01:11<00:38,  5.17it/s]
+ 65%|██████▌   | 369/567 [01:11<00:38,  5.15it/s]
+ 65%|██████▌   | 370/567 [01:11<00:38,  5.13it/s]
+ 65%|██████▌   | 371/567 [01:11<00:38,  5.12it/s]
+ 66%|██████▌   | 372/567 [01:11<00:37,  5.14it/s]
+ 66%|██████▌   | 373/567 [01:12<00:37,  5.16it/s]
+ 66%|██████▌   | 374/567 [01:12<00:37,  5.14it/s]
+ 66%|██████▌   | 375/567 [01:12<00:37,  5.15it/s]
+ 66%|██████▋   | 376/567 [01:12<00:37,  5.16it/s]
+ 66%|██████▋   | 377/567 [01:12<00:36,  5.14it/s]
+ 67%|██████▋   | 378/567 [01:12<00:36,  5.23it/s]
+ 67%|██████▋   | 379/567 [01:13<00:36,  5.14it/s]
+ 67%|██████▋   | 380/567 [01:13<00:36,  5.12it/s]
+ 67%|██████▋   | 381/567 [01:13<00:36,  5.14it/s]
+ 67%|██████▋   | 382/567 [01:13<00:35,  5.15it/s]
+ 68%|██████▊   | 383/567 [01:13<00:35,  5.14it/s]
+ 68%|██████▊   | 384/567 [01:14<00:35,  5.15it/s]
+ 68%|██████▊   | 385/567 [01:14<00:35,  5.14it/s]
+ 68%|██████▊   | 386/567 [01:14<00:35,  5.15it/s]
+ 68%|██████▊   | 387/567 [01:14<00:34,  5.16it/s]
+ 68%|██████▊   | 388/567 [01:14<00:34,  5.14it/s]
+ 69%|██████▊   | 389/567 [01:15<00:34,  5.16it/s]
+ 69%|██████▉   | 390/567 [01:15<00:34,  5.14it/s]
+ 69%|██████▉   | 391/567 [01:15<00:34,  5.13it/s]
+ 69%|██████▉   | 392/567 [01:15<00:34,  5.12it/s]
+ 69%|██████▉   | 393/567 [01:15<00:33,  5.14it/s]
+ 69%|██████▉   | 394/567 [01:16<00:33,  5.16it/s]
+ 70%|██████▉   | 395/567 [01:16<00:33,  5.17it/s]
+ 70%|██████▉   | 396/567 [01:16<00:33,  5.17it/s]
+ 70%|███████   | 397/567 [01:16<00:32,  5.18it/s]
+ 70%|███████   | 398/567 [01:16<00:32,  5.18it/s]
+ 70%|███████   | 399/567 [01:17<00:32,  5.15it/s]
+ 71%|███████   | 400/567 [01:17<00:32,  5.13it/s]
+ 71%|███████   | 401/567 [01:17<00:32,  5.15it/s]
+ 71%|███████   | 402/567 [01:17<00:32,  5.15it/s]
+ 71%|███████   | 403/567 [01:17<00:31,  5.17it/s]
+ 71%|███████▏  | 404/567 [01:18<00:31,  5.17it/s]
+ 71%|███████▏  | 405/567 [01:18<00:31,  5.17it/s]
+ 72%|███████▏  | 406/567 [01:18<00:31,  5.15it/s]
+ 72%|███████▏  | 407/567 [01:18<00:31,  5.16it/s]
+ 72%|███████▏  | 408/567 [01:18<00:30,  5.17it/s]
+ 72%|███████▏  | 409/567 [01:19<00:30,  5.17it/s]
+ 72%|███████▏  | 410/567 [01:19<00:30,  5.16it/s]
+ 72%|███████▏  | 411/567 [01:19<00:30,  5.16it/s]
+ 73%|███████▎  | 412/567 [01:19<00:29,  5.17it/s]
+ 73%|███████▎  | 413/567 [01:19<00:29,  5.18it/s]
+ 73%|███████▎  | 414/567 [01:19<00:29,  5.15it/s]
+ 73%|███████▎  | 415/567 [01:20<00:29,  5.16it/s]
+ 73%|███████▎  | 416/567 [01:20<00:29,  5.17it/s]
+ 74%|███████▎  | 417/567 [01:20<00:29,  5.15it/s]
+ 74%|███████▎  | 418/567 [01:20<00:29,  5.14it/s]
+ 74%|███████▍  | 419/567 [01:20<00:28,  5.16it/s]
+ 74%|███████▍  | 420/567 [01:21<00:28,  5.14it/s]
+ 74%|████��██▍  | 421/567 [01:21<00:28,  5.15it/s]
+ 74%|███████▍  | 422/567 [01:21<00:28,  5.13it/s]
+ 75%|███████▍  | 423/567 [01:21<00:27,  5.15it/s]
+ 75%|███████▍  | 424/567 [01:21<00:27,  5.17it/s]
+ 75%|███████▍  | 425/567 [01:22<00:27,  5.17it/s]
+ 75%|███████▌  | 426/567 [01:22<00:27,  5.18it/s]
+ 75%|███████▌  | 427/567 [01:22<00:27,  5.15it/s]
+ 75%|███████▌  | 428/567 [01:22<00:26,  5.16it/s]
+ 76%|███████▌  | 429/567 [01:22<00:26,  5.15it/s]
+ 76%|███████▌  | 430/567 [01:23<00:26,  5.16it/s]
+ 76%|███████▌  | 431/567 [01:23<00:26,  5.16it/s]
+ 76%|███████▌  | 432/567 [01:23<00:26,  5.15it/s]
+ 76%|███████▋  | 433/567 [01:23<00:26,  5.13it/s]
+ 77%|███████▋  | 434/567 [01:23<00:25,  5.12it/s]
+ 77%|███████▋  | 435/567 [01:24<00:25,  5.15it/s]
+ 77%|███████▋  | 436/567 [01:24<00:25,  5.16it/s]
+ 77%|███████▋  | 437/567 [01:24<00:25,  5.14it/s]
+ 77%|███████▋  | 438/567 [01:24<00:25,  5.15it/s]
+ 77%|███████▋  | 439/567 [01:24<00:24,  5.15it/s]
+ 78%|███████▊  | 440/567 [01:25<00:24,  5.14it/s]
+ 78%|███████▊  | 441/567 [01:25<00:24,  5.23it/s]
+ 78%|███████▊  | 442/567 [01:25<00:24,  5.14it/s]
+ 78%|███████▊  | 443/567 [01:25<00:24,  5.13it/s]
+ 78%|███████▊  | 444/567 [01:25<00:23,  5.15it/s]
+ 78%|███████▊  | 445/567 [01:26<00:23,  5.16it/s]
+ 79%|███████▊  | 446/567 [01:26<00:23,  5.16it/s]
+ 79%|███████▉  | 447/567 [01:26<00:23,  5.18it/s]
+ 79%|███████▉  | 448/567 [01:26<00:23,  5.17it/s]
+ 79%|███████▉  | 449/567 [01:26<00:22,  5.18it/s]
+ 79%|███████▉  | 450/567 [01:26<00:22,  5.19it/s]
+ 80%|███████▉  | 451/567 [01:27<00:22,  5.18it/s]
+ 80%|███████▉  | 452/567 [01:27<00:22,  5.19it/s]
+ 80%|███████▉  | 453/567 [01:27<00:22,  5.18it/s]
+ 80%|████████  | 454/567 [01:27<00:21,  5.17it/s]
+ 80%|████████  | 455/567 [01:27<00:21,  5.16it/s]
+ 80%|████████  | 456/567 [01:28<00:21,  5.17it/s]
+ 81%|████████  | 457/567 [01:28<00:21,  5.19it/s]
+ 81%|████████  | 458/567 [01:28<00:20,  5.20it/s]
+ 81%|████████  | 459/567 [01:28<00:20,  5.21it/s]
+ 81%|████████  | 460/567 [01:28<00:20,  5.21it/s]
+ 81%|████████▏ | 461/567 [01:29<00:20,  5.22it/s]
+ 81%|████████▏ | 462/567 [01:29<00:20,  5.19it/s]
+ 82%|████████▏ | 463/567 [01:29<00:20,  5.18it/s]
+ 82%|████████▏ | 464/567 [01:29<00:19,  5.19it/s]
+ 82%|████████▏ | 465/567 [01:29<00:19,  5.20it/s]
+ 82%|████████▏ | 466/567 [01:30<00:19,  5.20it/s]
+ 82%|████████▏ | 467/567 [01:30<00:19,  5.21it/s]
+ 83%|████████▎ | 468/567 [01:30<00:18,  5.21it/s]
+ 83%|████████▎ | 469/567 [01:30<00:18,  5.19it/s]
+ 83%|████████▎ | 470/567 [01:30<00:18,  5.20it/s]
+ 83%|████████▎ | 471/567 [01:31<00:18,  5.20it/s]
+ 83%|████████▎ | 472/567 [01:31<00:18,  5.21it/s]
+ 83%|████████▎ | 473/567 [01:31<00:18,  5.19it/s]
+ 84%|████████▎ | 474/567 [01:31<00:17,  5.20it/s]
+ 84%|████████▍ | 475/567 [01:31<00:17,  5.21it/s]
+ 84%|████████▍ | 476/567 [01:31<00:17,  5.21it/s]
+ 84%|████████▍ | 477/567 [01:32<00:17,  5.19it/s]
+ 84%|████████▍ | 478/567 [01:32<00:17,  5.20it/s]
+ 84%|████████▍ | 479/567 [01:32<00:16,  5.20it/s]
+ 85%|████████▍ | 480/567 [01:32<00:16,  5.18it/s]
+ 85%|████████▍ | 481/567 [01:32<00:16,  5.17it/s]
+ 85%|████████▌ | 482/567 [01:33<00:16,  5.18it/s]
+ 85%|████████▌ | 483/567 [01:33<00:16,  5.17it/s]
+ 85%|████████▌ | 484/567 [01:33<00:15,  5.19it/s]
+ 86%|████████▌ | 485/567 [01:33<00:15,  5.17it/s]
+ 86%|████████▌ | 486/567 [01:33<00:15,  5.19it/s]
+ 86%|████████▌ | 487/567 [01:34<00:15,  5.20it/s]
+ 86%|████████▌ | 488/567 [01:34<00:15,  5.21it/s]
+ 86%|████████▌ | 489/567 [01:34<00:14,  5.21it/s]
+ 86%|████████▋ | 490/567 [01:34<00:14,  5.19it/s]
+ 87%|████████▋ | 491/567 [01:34<00:14,  5.20it/s]
+ 87%|████████▋ | 492/567 [01:35<00:14,  5.18it/s]
+ 87%|████████▋ | 493/567 [01:35<00:14,  5.20it/s]
+ 87%|████████▋ | 494/567 [01:35<00:14,  5.21it/s]
+ 87%|████████▋ | 495/567 [01:35<00:13,  5.19it/s]
+ 87%|████████▋ | 496/567 [01:35<00:13,  5.17it/s]
+ 88%|████████▊ | 497/567 [01:36<00:13,  5.16it/s]
+ 88%|████████▊ | 498/567 [01:36<00:13,  5.19it/s]
+ 88%|████████▊ | 499/567 [01:36<00:13,  5.20it/s]
+ 88%|████████▊ | 500/567 [01:36<00:12,  5.18it/s]
+ 88%|████████▊ | 501/567 [01:36<00:12,  5.19it/s]
+ 89%|████████▊ | 502/567 [01:36<00:12,  5.20it/s]
+ 89%|████████▊ | 503/567 [01:37<00:12,  5.18it/s]
+ 89%|████████▉ | 504/567 [01:37<00:11,  5.26it/s]
+ 89%|████████▉ | 505/567 [01:37<00:11,  5.18it/s]
+ 89%|████████▉ | 506/567 [01:37<00:11,  5.17it/s]
+ 89%|████████▉ | 507/567 [01:37<00:11,  5.19it/s]
+ 90%|████████▉ | 508/567 [01:38<00:11,  5.20it/s]
+ 90%|████████▉ | 509/567 [01:38<00:11,  5.18it/s]
+ 90%|████████▉ | 510/567 [01:38<00:10,  5.19it/s]
+ 90%|█████████ | 511/567 [01:38<00:10,  5.16it/s]
+ 90%|█████████ | 512/567 [01:38<00:10,  5.17it/s]
+ 90%|█████████ | 513/567 [01:39<00:10,  5.18it/s]
+ 91%|█████████ | 514/567 [01:39<00:10,  5.15it/s]
+ 91%|█████████ | 515/567 [01:39<00:10,  5.16it/s]
+ 91%|█████████ | 516/567 [01:39<00:09,  5.15it/s]
+ 91%|█████████ | 517/567 [01:39<00:09,  5.13it/s]
+ 91%|█████████▏| 518/567 [01:40<00:09,  5.13it/s]
+ 92%|█████████▏| 519/567 [01:40<00:09,  5.15it/s]
+ 92%|█████████▏| 520/567 [01:40<00:09,  5.16it/s]
+ 92%|█████████▏| 521/567 [01:40<00:08,  5.17it/s]
+ 92%|█████████▏| 522/567 [01:40<00:08,  5.18it/s]
+ 92%|█████████▏| 523/567 [01:41<00:08,  5.18it/s]
+ 92%|█████████▏| 524/567 [01:41<00:08,  5.18it/s]
+ 93%|█████████▎| 525/567 [01:41<00:08,  5.16it/s]
+ 93%|█████████▎| 526/567 [01:41<00:07,  5.14it/s]
+ 93%|█████████▎| 527/567 [01:41<00:07,  5.15it/s]
+ 93%|█████████▎| 528/567 [01:42<00:07,  5.16it/s]
+ 93%|█████████▎| 529/567 [01:42<00:07,  5.17it/s]
+ 93%|█████████▎| 530/567 [01:42<00:07,  5.17it/s]
+ 94%|█████████▎| 531/567 [01:42<00:06,  5.18it/s]
+ 94%|█████████▍| 532/567 [01:42<00:06,  5.16it/s]
+ 94%|█████████▍| 533/567 [01:42<00:06,  5.16it/s]
+ 94%|█████████▍| 534/567 [01:43<00:06,  5.17it/s]
+ 94%|█████████▍| 535/567 [01:43<00:06,  5.18it/s]
+ 95%|█████████▍| 536/567 [01:43<00:06,  5.16it/s]
+ 95%|█████████▍| 537/567 [01:43<00:05,  5.16it/s]
+ 95%|█████████▍| 538/567 [01:43<00:05,  5.17it/s]
+ 95%|█████████▌| 539/567 [01:44<00:05,  5.18it/s]
+ 95%|█████████▌| 540/567 [01:44<00:05,  5.16it/s]
+ 95%|█████████▌| 541/567 [01:44<00:05,  5.17it/s]
+ 96%|█████████▌| 542/567 [01:44<00:04,  5.18it/s]
+ 96%|█████████▌| 543/567 [01:44<00:04,  5.15it/s]
+ 96%|█████████▌| 544/567 [01:45<00:04,  5.14it/s]
+ 96%|█████████▌| 545/567 [01:45<00:04,  5.15it/s]
+ 96%|█████████▋| 546/567 [01:45<00:04,  5.14it/s]
+ 96%|█████████▋| 547/567 [01:45<00:03,  5.16it/s]
+ 97%|█████████▋| 548/567 [01:45<00:03,  5.14it/s]
+ 97%|█████████▋| 549/567 [01:46<00:03,  5.15it/s]
+ 97%|█████████▋| 550/567 [01:46<00:03,  5.16it/s]
+ 97%|█████████▋| 551/567 [01:46<00:03,  5.17it/s]
+ 97%|█████████▋| 552/567 [01:46<00:02,  5.18it/s]
+ 98%|█████████▊| 553/567 [01:46<00:02,  5.15it/s]
+ 98%|█████████▊| 554/567 [01:47<00:02,  5.16it/s]
+ 98%|█████████▊| 555/567 [01:47<00:02,  5.14it/s]
+ 98%|█████████▊| 556/567 [01:47<00:02,  5.15it/s]
+ 98%|█████████▊| 557/567 [01:47<00:01,  5.17it/s]
+ 98%|█████████▊| 558/567 [01:47<00:01,  5.15it/s]
+ 99%|█████████▊| 559/567 [01:48<00:01,  5.13it/s]
+ 99%|█████████▉| 560/567 [01:48<00:01,  5.12it/s]
+ 99%|█████████▉| 561/567 [01:48<00:01,  5.14it/s]
+ 99%|█████████▉| 562/567 [01:48<00:00,  5.16it/s]
+ 99%|█████████▉| 563/567 [01:48<00:00,  5.14it/s]
+ 99%|█████████▉| 564/567 [01:48<00:00,  5.16it/s]
+100%|█████████▉| 565/567 [01:49<00:00,  5.18it/s]
+100%|█████████▉| 566/567 [01:49<00:00,  5.17it/s]
+100%|██████████| 567/567 [01:49<00:00,  5.25it/s]
+100%|██████████| 567/567 [01:49<00:00,  5.18it/s]
+Saving model to directory step10000
+[2026-07-22 13:43:38,837] [INFO] [logging.py:123:log_dist] [Rank 0] [Torch] Checkpoint global_step10000 is about to be saved!
+[2026-07-22 13:43:38,861] [INFO] [logging.py:123:log_dist] [Rank 0] Saving model checkpoint: /content/anima_campaign/runs/e017_mb3_s0/20260722_11-56-14/global_step10000/mp_rank_00_model_states.pt
+TRAINING COMPLETE!
+[rank0]:[W722 13:43:40.079387810 ProcessGroupNCCL.cpp:1575] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
+[2026-07-22 13:43:46,519] [INFO] [launch.py:367:main] Process 27300 exits successfully.