Buckets:
| Running 1 job | |
| { | |
| "type": "diffusion_trainer", | |
| "training_folder": "/app/ai-toolkit/output", | |
| "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", | |
| "device": "cuda", | |
| "trigger_word": null, | |
| "performance_log_every": 10, | |
| "network": { | |
| "type": "lora", | |
| "linear": 32, | |
| "linear_alpha": 32, | |
| "conv": 16, | |
| "conv_alpha": 16, | |
| "lokr_full_rank": true, | |
| "lokr_factor": -1, | |
| "network_kwargs": { | |
| "ignore_if_contains": [] | |
| } | |
| }, | |
| "save": { | |
| "dtype": "bf16", | |
| "save_every": 25, | |
| "max_step_saves_to_keep": 500, | |
| "save_format": "diffusers", | |
| "push_to_hub": false | |
| }, | |
| "datasets": [ | |
| { | |
| "folder_path": "/app/ai-toolkit/datasets/nsfw", | |
| "mask_path": null, | |
| "mask_min_value": 0.1, | |
| "default_caption": "", | |
| "caption_ext": "txt", | |
| "caption_dropout_rate": 0.05, | |
| "cache_latents_to_disk": false, | |
| "is_reg": false, | |
| "network_weight": 1, | |
| "resolution": [ | |
| 512, | |
| 256 | |
| ], | |
| "controls": [], | |
| "shrink_video_to_frames": true, | |
| "num_frames": 1, | |
| "flip_x": false, | |
| "flip_y": false, | |
| "num_repeats": 1 | |
| } | |
| ], | |
| "train": { | |
| "batch_size": 7, | |
| "bypass_guidance_embedding": false, | |
| "steps": 20000, | |
| "gradient_accumulation": 1, | |
| "train_unet": true, | |
| "train_text_encoder": false, | |
| "gradient_checkpointing": true, | |
| "noise_scheduler": "ddpm", | |
| "optimizer": "adamw8bit", | |
| "timestep_type": "sigmoid", | |
| "content_or_style": "balanced", | |
| "optimizer_params": { | |
| "weight_decay": 0.0001 | |
| }, | |
| "unload_text_encoder": false, | |
| "cache_text_embeddings": false, | |
| "lr": 7e-05, | |
| "ema_config": { | |
| "use_ema": false, | |
| "ema_decay": 0.99 | |
| }, | |
| "skip_first_sample": false, | |
| "force_first_sample": false, | |
| "disable_sampling": true, | |
| "dtype": "bf16", | |
| "diff_output_preservation": false, | |
| "diff_output_preservation_multiplier": 1, | |
| "diff_output_preservation_class": "person", | |
| "switch_boundary_every": 1, | |
| "loss_type": "mse", | |
| "do_differential_guidance": true, | |
| "differential_guidance_scale": 3 | |
| }, | |
| "logging": { | |
| "log_every": 1, | |
| "use_ui_logger": true | |
| }, | |
| "model": { | |
| "name_or_path": "stable-diffusion-v1-5/stable-diffusion-v1-5", | |
| "quantize": false, | |
| "qtype": "qfloat8", | |
| "quantize_te": false, | |
| "qtype_te": "qfloat8", | |
| "arch": "sd1", | |
| "low_vram": false, | |
| "model_kwargs": {}, | |
| "compile": false | |
| }, | |
| "sample": { | |
| "sampler": "ddpm", | |
| "sample_every": 50, | |
| "sample_start_step": 5000, | |
| "width": 512, | |
| "height": 512, | |
| "samples": [], | |
| "neg": "", | |
| "seed": 42, | |
| "walk_seed": true, | |
| "guidance_scale": 6, | |
| "sample_steps": 30, | |
| "num_frames": 1, | |
| "fps": 1 | |
| } | |
| } | |
| Using SQLite database at /app/ai-toolkit/aitk_db.db | |
| Job ID: "b189da96-babe-4a7c-846a-ad0aa5ac67e8" | |
| ############################################# | |
| # Running job: jam | |
| ############################################# | |
| Running 1 process | |
| Loading pipeline components...: 0%| | 0/6 [00:00<?, ?it/s] | |
| Loading weights: 0%| | 0/196 [00:00<?, ?it/s][A Loading weights: 100%|██████████| 196/196 [00:00<00:00, 11144.78it/s] | |
| Loading pipeline components...: 67%|██████▋ | 4/6 [00:00<00:00, 32.38it/s] Loading pipeline components...: 100%|██████████| 6/6 [00:00<00:00, 37.55it/s] | |
| create LoRA network. base dim (rank): 32, alpha: 32 | |
| neuron dropout: p=None, rank dropout: p=None, module dropout: p=None | |
| apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16 | |
| create LoRA for Text Encoder: 0 modules. | |
| create LoRA for U-Net: 278 modules. | |
| enable LoRA for U-Net | |
| Dataset: /app/ai-toolkit/datasets/nsfw | |
| - Preprocessing image dimensions | |
| 0%| | 0/2672 [00:00<?, ?it/s] 3%|▎ | 88/2672 [00:00<00:02, 867.72it/s] 7%|▋ | 180/2672 [00:00<00:02, 893.87it/s] 10%|█ | 276/2672 [00:00<00:02, 922.59it/s] 14%|█▍ | 369/2672 [00:00<00:02, 905.98it/s] 17%|█▋ | 460/2672 [00:00<00:02, 898.34it/s] 21%|██ | 550/2672 [00:00<00:02, 880.64it/s] 24%|██▍ | 643/2672 [00:00<00:02, 894.60it/s] 27%|██▋ | 733/2672 [00:00<00:02, 893.46it/s] 31%|███ | 834/2672 [00:00<00:01, 927.41it/s] 35%|███▍ | 927/2672 [00:01<00:01, 900.90it/s] 38%|███▊ | 1019/2672 [00:01<00:01, 906.21it/s] 42%|████▏ | 1110/2672 [00:01<00:01, 896.80it/s] 45%|████▌ | 1203/2672 [00:01<00:01, 904.92it/s] 49%|████▊ | 1298/2672 [00:01<00:01, 918.07it/s] 52%|█████▏ | 1391/2672 [00:01<00:01, 919.55it/s] 56%|█████▌ | 1484/2672 [00:01<00:01, 903.45it/s] 59%|█████▉ | 1575/2672 [00:01<00:01, 893.51it/s] 62%|██████▏ | 1665/2672 [00:01<00:01, 879.39it/s] 66%|██████▌ | 1757/2672 [00:01<00:01, 887.08it/s] 69%|██████▉ | 1847/2672 [00:02<00:00, 890.73it/s] 73%|███████▎ | 1940/2672 [00:02<00:00, 898.16it/s] 76%|███████▌ | 2030/2672 [00:02<00:00, 885.90it/s] 79%|███████▉ | 2119/2672 [00:02<00:00, 880.25it/s] 83%|████████▎ | 2213/2672 [00:02<00:00, 897.52it/s] 86%|████████▌ | 2303/2672 [00:02<00:00, 885.34it/s] 90%|████████▉ | 2392/2672 [00:02<00:00, 878.94it/s] 93%|█████████▎| 2486/2672 [00:02<00:00, 893.47it/s] 96%|█████████▋| 2576/2672 [00:02<00:00, 892.54it/s] 100%|█████████▉| 2667/2672 [00:02<00:00, 897.10it/s] 100%|██████████| 2672/2672 [00:02<00:00, 897.07it/s] | |
| - Found 2672 images | |
| Bucket sizes for /app/ai-toolkit/datasets/nsfw: | |
| 416x624: 1801 files | |
| 464x304: 96 files | |
| 448x256: 24 files | |
| 448x304: 133 files | |
| 576x448: 45 files | |
| 624x416: 349 files | |
| 672x384: 66 files | |
| 448x576: 78 files | |
| 448x352: 22 files | |
| 464x256: 38 files | |
| 384x672: 3 files | |
| 464x368: 5 files | |
| 592x432: 2 files | |
| 432x592: 2 files | |
| 464x560: 1 files | |
| 464x288: 4 files | |
| 16x16: 1 files | |
| 560x464: 1 files | |
| 464x464: 1 files | |
| 19 buckets made | |
| Dataset: /app/ai-toolkit/datasets/nsfw | |
| - Preprocessing image dimensions | |
| 0%| | 0/2672 [00:00<?, ?it/s] 100%|██████████| 2672/2672 [00:00<00:00, 82410.59it/s] | |
| - Found 2672 images | |
| Bucket sizes for /app/ai-toolkit/datasets/nsfw: | |
| 208x304: 1801 files | |
| 304x208: 578 files | |
| 336x192: 132 files | |
| 288x224: 69 files | |
| 224x288: 80 files | |
| 192x336: 3 files | |
| 272x240: 6 files | |
| 240x272: 1 files | |
| 16x16: 1 files | |
| 256x256: 1 files | |
| 10 buckets made | |
| Skipping first sample due to config setting | |
| jam: 0%| | 0/20000 [00:00<?, ?it/s] jam: 0%| | 0/20000 [00:00<?, ?it/s, lr: 7.0e-05 loss: 1.834e-01] jam: 0%| | 1/20000 [00:00<4:46:17, 1.16it/s, lr: 7.0e-05 loss: 1.834e-01] jam: 0%| | 1/20000 [00:01<4:46:17, 1.16it/s, lr: 7.0e-05 loss: 9.410e-02] jam: 0%| | 2/20000 [00:01<2:42:05, 2.06it/s, lr: 7.0e-05 loss: 9.410e-02] jam: 0%| | 2/20000 [00:01<2:42:05, 2.06it/s, lr: 7.0e-05 loss: 1.203e-01] jam: 0%| | 3/20000 [00:01<2:15:13, 2.46it/s, lr: 7.0e-05 loss: 1.203e-01] jam: 0%| | 3/20000 [00:01<2:15:13, 2.46it/s, lr: 7.0e-05 loss: 2.325e-01] jam: 0%| | 4/20000 [00:01<1:59:40, 2.78it/s, lr: 7.0e-05 loss: 2.325e-01] jam: 0%| | 4/20000 [00:02<1:59:40, 2.78it/s, lr: 7.0e-05 loss: 1.513e-01] jam: 0%| | 5/20000 [00:02<1:55:33, 2.88it/s, lr: 7.0e-05 loss: 1.513e-01] jam: 0%| | 5/20000 [00:02<1:55:33, 2.88it/s, lr: 7.0e-05 loss: 2.363e-01] jam: 0%| | 6/20000 [00:02<1:25:58, 3.88it/s, lr: 7.0e-05 loss: 2.363e-01] jam: 0%| | 6/20000 [00:02<1:25:58, 3.88it/s, lr: 7.0e-05 loss: 7.502e-02] jam: 0%| | 7/20000 [00:02<2:02:08, 2.73it/s, lr: 7.0e-05 loss: 7.502e-02] jam: 0%| | 7/20000 [00:03<2:02:08, 2.73it/s, lr: 7.0e-05 loss: 2.047e-01] jam: 0%| | 8/20000 [00:03<2:00:20, 2.77it/s, lr: 7.0e-05 loss: 2.047e-01] jam: 0%| | 8/20000 [00:03<2:00:20, 2.77it/s, lr: 7.0e-05 loss: 1.000e-01] jam: 0%| | 9/20000 [00:03<1:52:27, 2.96it/s, lr: 7.0e-05 loss: 1.000e-01] jam: 0%| | 9/20000 [00:03<1:52:27, 2.96it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 0%| | 10/20000 [00:03<2:02:28, 2.72it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 0%| | 10/20000 [00:04<2:02:28, 2.72it/s, lr: 7.0e-05 loss: 2.088e-01] jam: 0%| | 11/20000 [00:04<1:27:13, 3.82it/s, lr: 7.0e-05 loss: 2.088e-01] jam: 0%| | 11/20000 [00:04<1:27:13, 3.82it/s, lr: 7.0e-05 loss: 1.814e-01] jam: 0%| | 12/20000 [00:04<1:56:28, 2.86it/s, lr: 7.0e-05 loss: 1.814e-01] jam: 0%| | 12/20000 [00:04<1:56:28, 2.86it/s, lr: 7.0e-05 loss: 2.284e-02] jam: 0%| | 13/20000 [00:04<1:58:43, 2.81it/s, lr: 7.0e-05 loss: 2.284e-02] jam: 0%| | 13/20000 [00:05<1:58:43, 2.81it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 0%| | 14/20000 [00:05<1:54:24, 2.91it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 0%| | 14/20000 [00:05<1:54:24, 2.91it/s, lr: 7.0e-05 loss: 6.063e-02] jam: 0%| | 15/20000 [00:05<1:58:39, 2.81it/s, lr: 7.0e-05 loss: 6.063e-02] jam: 0%| | 15/20000 [00:05<1:58:39, 2.81it/s, lr: 7.0e-05 loss: 1.324e-01] jam: 0%| | 16/20000 [00:05<1:55:49, 2.88it/s, lr: 7.0e-05 loss: 1.324e-01] jam: 0%| | 16/20000 [00:06<1:55:49, 2.88it/s, lr: 7.0e-05 loss: 2.155e-01] jam: 0%| | 17/20000 [00:06<1:25:30, 3.89it/s, lr: 7.0e-05 loss: 2.155e-01] jam: 0%| | 17/20000 [00:06<1:25:30, 3.89it/s, lr: 7.0e-05 loss: 1.941e-01] jam: 0%| | 18/20000 [00:06<1:15:16, 4.42it/s, lr: 7.0e-05 loss: 1.941e-01] jam: 0%| | 18/20000 [00:06<1:15:16, 4.42it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 0%| | 19/20000 [00:06<1:44:51, 3.18it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 0%| | 19/20000 [00:07<1:44:51, 3.18it/s, lr: 7.0e-05 loss: 1.430e-01] jam: 0%| | 20/20000 [00:07<1:25:29, 3.89it/s, lr: 7.0e-05 loss: 1.430e-01] jam: 0%| | 20/20000 [00:07<1:25:29, 3.89it/s, lr: 7.0e-05 loss: 1.177e-01] jam: 0%| | 21/20000 [00:07<1:19:21, 4.20it/s, lr: 7.0e-05 loss: 1.177e-01] jam: 0%| | 21/20000 [00:07<1:19:21, 4.20it/s, lr: 7.0e-05 loss: 2.006e-01] jam: 0%| | 22/20000 [00:07<1:08:55, 4.83it/s, lr: 7.0e-05 loss: 2.006e-01] jam: 0%| | 22/20000 [00:07<1:08:55, 4.83it/s, lr: 7.0e-05 loss: 5.377e-02] jam: 0%| | 23/20000 [00:07<1:40:37, 3.31it/s, lr: 7.0e-05 loss: 5.377e-02] jam: 0%| | 23/20000 [00:08<1:40:37, 3.31it/s, lr: 7.0e-05 loss: 1.024e-01] jam: 0%| | 24/20000 [00:08<1:23:11, 4.00it/s, lr: 7.0e-05 loss: 1.024e-01] jam: 0%| | 24/20000 [00:08<1:23:11, 4.00it/s, lr: 7.0e-05 loss: 1.175e-01] jam: 0%| | 25/20000 [00:08<1:50:43, 3.01it/s, lr: 7.0e-05 loss: 1.175e-01] jam: 0%| | 25/20000 [00:09<1:50:43, 3.01it/s, lr: 7.0e-05 loss: 2.082e-01] | |
| Saving at step 25 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000025.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 0%| | 26/20000 [00:09<1:50:43, 3.01it/s, lr: 7.0e-05 loss: 1.044e-01] jam: 0%| | 27/20000 [00:09<1:45:22, 3.16it/s, lr: 7.0e-05 loss: 1.044e-01] jam: 0%| | 27/20000 [00:09<1:45:22, 3.16it/s, lr: 7.0e-05 loss: 3.375e-01] jam: 0%| | 28/20000 [00:09<1:17:58, 4.27it/s, lr: 7.0e-05 loss: 3.375e-01] jam: 0%| | 28/20000 [00:09<1:17:58, 4.27it/s, lr: 7.0e-05 loss: 1.376e-01] jam: 0%| | 29/20000 [00:09<1:06:25, 5.01it/s, lr: 7.0e-05 loss: 1.376e-01] jam: 0%| | 29/20000 [00:10<1:06:25, 5.01it/s, lr: 7.0e-05 loss: 2.666e-01] jam: 0%| | 30/20000 [00:10<1:03:17, 5.26it/s, lr: 7.0e-05 loss: 2.666e-01] jam: 0%| | 30/20000 [00:10<1:03:17, 5.26it/s, lr: 7.0e-05 loss: 2.210e-01] jam: 0%| | 31/20000 [00:10<1:05:46, 5.06it/s, lr: 7.0e-05 loss: 2.210e-01] jam: 0%| | 31/20000 [00:10<1:05:46, 5.06it/s, lr: 7.0e-05 loss: 9.065e-02] jam: 0%| | 32/20000 [00:10<1:23:58, 3.96it/s, lr: 7.0e-05 loss: 9.065e-02] jam: 0%| | 32/20000 [00:11<1:23:58, 3.96it/s, lr: 7.0e-05 loss: 6.194e-02] jam: 0%| | 33/20000 [00:11<1:51:57, 2.97it/s, lr: 7.0e-05 loss: 6.194e-02] jam: 0%| | 33/20000 [00:11<1:51:57, 2.97it/s, lr: 7.0e-05 loss: 2.095e-01] jam: 0%| | 34/20000 [00:11<1:24:32, 3.94it/s, lr: 7.0e-05 loss: 2.095e-01] jam: 0%| | 34/20000 [00:11<1:24:32, 3.94it/s, lr: 7.0e-05 loss: 9.711e-02] jam: 0%| | 35/20000 [00:11<1:52:59, 2.94it/s, lr: 7.0e-05 loss: 9.711e-02] jam: 0%| | 35/20000 [00:11<1:52:59, 2.94it/s, lr: 7.0e-05 loss: 2.781e-01] jam: 0%| | 36/20000 [00:11<1:52:57, 2.95it/s, lr: 7.0e-05 loss: 2.781e-01] jam: 0%| | 36/20000 [00:12<1:52:57, 2.95it/s, lr: 7.0e-05 loss: 1.459e-01] jam: 0%| | 37/20000 [00:12<1:58:11, 2.81it/s, lr: 7.0e-05 loss: 1.459e-01] jam: 0%| | 37/20000 [00:12<1:58:11, 2.81it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 0%| | 38/20000 [00:12<1:13:35, 4.52it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 0%| | 38/20000 [00:12<1:13:35, 4.52it/s, lr: 7.0e-05 loss: 1.023e-01] jam: 0%| | 39/20000 [00:12<1:51:49, 2.97it/s, lr: 7.0e-05 loss: 1.023e-01] jam: 0%| | 39/20000 [00:13<1:51:49, 2.97it/s, lr: 7.0e-05 loss: 1.824e-01] jam: 0%| | 40/20000 [00:13<1:21:07, 4.10it/s, lr: 7.0e-05 loss: 1.824e-01] jam: 0%| | 40/20000 [00:13<1:21:07, 4.10it/s, lr: 7.0e-05 loss: 9.918e-02] jam: 0%| | 41/20000 [00:13<1:45:58, 3.14it/s, lr: 7.0e-05 loss: 9.918e-02] jam: 0%| | 41/20000 [00:13<1:45:58, 3.14it/s, lr: 7.0e-05 loss: 1.571e-01] jam: 0%| | 42/20000 [00:13<1:58:54, 2.80it/s, lr: 7.0e-05 loss: 1.571e-01] jam: 0%| | 42/20000 [00:14<1:58:54, 2.80it/s, lr: 7.0e-05 loss: 2.596e-02] jam: 0%| | 43/20000 [00:14<1:45:49, 3.14it/s, lr: 7.0e-05 loss: 2.596e-02] jam: 0%| | 43/20000 [00:14<1:45:49, 3.14it/s, lr: 7.0e-05 loss: 1.935e-01] jam: 0%| | 44/20000 [00:14<1:52:16, 2.96it/s, lr: 7.0e-05 loss: 1.935e-01] jam: 0%| | 44/20000 [00:14<1:52:16, 2.96it/s, lr: 7.0e-05 loss: 1.340e-01] jam: 0%| | 45/20000 [00:14<1:55:55, 2.87it/s, lr: 7.0e-05 loss: 1.340e-01] jam: 0%| | 45/20000 [00:15<1:55:55, 2.87it/s, lr: 7.0e-05 loss: 1.125e-01] jam: 0%| | 46/20000 [00:15<1:32:03, 3.61it/s, lr: 7.0e-05 loss: 1.125e-01] jam: 0%| | 46/20000 [00:15<1:32:03, 3.61it/s, lr: 7.0e-05 loss: 1.160e-01] jam: 0%| | 47/20000 [00:15<1:17:57, 4.27it/s, lr: 7.0e-05 loss: 1.160e-01] jam: 0%| | 47/20000 [00:15<1:17:57, 4.27it/s, lr: 7.0e-05 loss: 1.037e-01] jam: 0%| | 48/20000 [00:15<1:20:34, 4.13it/s, lr: 7.0e-05 loss: 1.037e-01] jam: 0%| | 48/20000 [00:15<1:20:34, 4.13it/s, lr: 7.0e-05 loss: 1.931e-01] jam: 0%| | 49/20000 [00:15<1:44:11, 3.19it/s, lr: 7.0e-05 loss: 1.931e-01] jam: 0%| | 49/20000 [00:16<1:44:11, 3.19it/s, lr: 7.0e-05 loss: 1.647e-01] jam: 0%| | 50/20000 [00:16<1:52:11, 2.96it/s, lr: 7.0e-05 loss: 1.647e-01] jam: 0%| | 50/20000 [00:16<1:52:11, 2.96it/s, lr: 7.0e-05 loss: 1.300e-01] | |
| Saving at step 50 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000050.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 0%| | 51/20000 [00:17<1:52:10, 2.96it/s, lr: 7.0e-05 loss: 4.610e-02] jam: 0%| | 52/20000 [00:17<1:44:48, 3.17it/s, lr: 7.0e-05 loss: 4.610e-02] jam: 0%| | 52/20000 [00:17<1:44:48, 3.17it/s, lr: 7.0e-05 loss: 1.043e-01] jam: 0%| | 53/20000 [00:17<1:56:28, 2.85it/s, lr: 7.0e-05 loss: 1.043e-01] jam: 0%| | 53/20000 [00:17<1:56:28, 2.85it/s, lr: 7.0e-05 loss: 1.956e-01] jam: 0%| | 54/20000 [00:17<1:19:04, 4.20it/s, lr: 7.0e-05 loss: 1.956e-01] jam: 0%| | 54/20000 [00:18<1:19:04, 4.20it/s, lr: 7.0e-05 loss: 2.260e-01] jam: 0%| | 55/20000 [00:18<1:09:45, 4.77it/s, lr: 7.0e-05 loss: 2.260e-01] jam: 0%| | 55/20000 [00:18<1:09:45, 4.77it/s, lr: 7.0e-05 loss: 6.328e-02] jam: 0%| | 56/20000 [00:18<1:01:07, 5.44it/s, lr: 7.0e-05 loss: 6.328e-02] jam: 0%| | 56/20000 [00:18<1:01:07, 5.44it/s, lr: 7.0e-05 loss: 1.255e-01] jam: 0%| | 57/20000 [00:18<1:04:29, 5.15it/s, lr: 7.0e-05 loss: 1.255e-01] jam: 0%| | 57/20000 [00:18<1:04:29, 5.15it/s, lr: 7.0e-05 loss: 9.383e-02] jam: 0%| | 58/20000 [00:18<1:41:15, 3.28it/s, lr: 7.0e-05 loss: 9.383e-02] jam: 0%| | 58/20000 [00:18<1:41:15, 3.28it/s, lr: 7.0e-05 loss: 1.523e-01] jam: 0%| | 59/20000 [00:18<1:14:13, 4.48it/s, lr: 7.0e-05 loss: 1.523e-01] jam: 0%| | 59/20000 [00:19<1:14:13, 4.48it/s, lr: 7.0e-05 loss: 1.073e-01] jam: 0%| | 60/20000 [00:19<1:41:07, 3.29it/s, lr: 7.0e-05 loss: 1.073e-01] jam: 0%| | 60/20000 [00:19<1:41:07, 3.29it/s, lr: 7.0e-05 loss: 1.566e-01] jam: 0%| | 61/20000 [00:19<1:14:44, 4.45it/s, lr: 7.0e-05 loss: 1.566e-01] jam: 0%| | 61/20000 [00:19<1:14:44, 4.45it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 0%| | 62/20000 [00:19<59:56, 5.54it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 0%| | 62/20000 [00:20<59:56, 5.54it/s, lr: 7.0e-05 loss: 1.319e-01] jam: 0%| | 63/20000 [00:20<1:37:56, 3.39it/s, lr: 7.0e-05 loss: 1.319e-01] jam: 0%| | 63/20000 [00:20<1:37:56, 3.39it/s, lr: 7.0e-05 loss: 1.462e-01] jam: 0%| | 64/20000 [00:20<1:13:48, 4.50it/s, lr: 7.0e-05 loss: 1.462e-01] jam: 0%| | 64/20000 [00:20<1:13:48, 4.50it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 0%| | 65/20000 [00:20<1:21:58, 4.05it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 0%| | 65/20000 [00:20<1:21:58, 4.05it/s, lr: 7.0e-05 loss: 1.306e-01] jam: 0%| | 66/20000 [00:20<1:07:23, 4.93it/s, lr: 7.0e-05 loss: 1.306e-01] jam: 0%| | 66/20000 [00:20<1:07:23, 4.93it/s, lr: 7.0e-05 loss: 2.812e-01] jam: 0%| | 67/20000 [00:20<1:09:09, 4.80it/s, lr: 7.0e-05 loss: 2.812e-01] jam: 0%| | 67/20000 [00:21<1:09:09, 4.80it/s, lr: 7.0e-05 loss: 6.227e-02] jam: 0%| | 68/20000 [00:21<1:43:57, 3.20it/s, lr: 7.0e-05 loss: 6.227e-02] jam: 0%| | 68/20000 [00:21<1:43:57, 3.20it/s, lr: 7.0e-05 loss: 1.625e-01] jam: 0%| | 69/20000 [00:21<1:20:38, 4.12it/s, lr: 7.0e-05 loss: 1.625e-01] jam: 0%| | 69/20000 [00:21<1:20:38, 4.12it/s, lr: 7.0e-05 loss: 5.593e-02] jam: 0%| | 70/20000 [00:21<1:14:42, 4.45it/s, lr: 7.0e-05 loss: 5.593e-02] jam: 0%| | 70/20000 [00:21<1:14:42, 4.45it/s, lr: 7.0e-05 loss: 1.648e-01] jam: 0%| | 71/20000 [00:21<1:14:45, 4.44it/s, lr: 7.0e-05 loss: 1.648e-01] jam: 0%| | 71/20000 [00:22<1:14:45, 4.44it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 0%| | 72/20000 [00:22<1:45:11, 3.16it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 0%| | 72/20000 [00:22<1:45:11, 3.16it/s, lr: 7.0e-05 loss: 4.629e-02] jam: 0%| | 73/20000 [00:22<1:22:31, 4.02it/s, lr: 7.0e-05 loss: 4.629e-02] jam: 0%| | 73/20000 [00:22<1:22:31, 4.02it/s, lr: 7.0e-05 loss: 5.453e-02] jam: 0%| | 74/20000 [00:22<1:45:16, 3.15it/s, lr: 7.0e-05 loss: 5.453e-02] jam: 0%| | 74/20000 [00:22<1:45:16, 3.15it/s, lr: 7.0e-05 loss: 1.321e-01] jam: 0%| | 75/20000 [00:22<1:13:52, 4.50it/s, lr: 7.0e-05 loss: 1.321e-01] jam: 0%| | 75/20000 [00:23<1:13:52, 4.50it/s, lr: 7.0e-05 loss: 2.056e-01] | |
| Saving at step 75 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000075.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 0%| | 76/20000 [00:24<1:13:52, 4.50it/s, lr: 7.0e-05 loss: 1.519e-01] jam: 0%| | 77/20000 [00:24<1:37:58, 3.39it/s, lr: 7.0e-05 loss: 1.519e-01] jam: 0%| | 77/20000 [00:24<1:37:58, 3.39it/s, lr: 7.0e-05 loss: 1.335e-01] jam: 0%| | 78/20000 [00:24<1:52:13, 2.96it/s, lr: 7.0e-05 loss: 1.335e-01] jam: 0%| | 78/20000 [00:24<1:52:13, 2.96it/s, lr: 7.0e-05 loss: 5.356e-02] jam: 0%| | 79/20000 [00:24<1:52:16, 2.96it/s, lr: 7.0e-05 loss: 5.356e-02] jam: 0%| | 79/20000 [00:25<1:52:16, 2.96it/s, lr: 7.0e-05 loss: 1.111e-01] jam: 0%| | 80/20000 [00:25<1:27:42, 3.79it/s, lr: 7.0e-05 loss: 1.111e-01] jam: 0%| | 80/20000 [00:25<1:27:42, 3.79it/s, lr: 7.0e-05 loss: 3.895e-01] jam: 0%| | 81/20000 [00:25<1:15:22, 4.40it/s, lr: 7.0e-05 loss: 3.895e-01] jam: 0%| | 81/20000 [00:25<1:15:22, 4.40it/s, lr: 7.0e-05 loss: 7.817e-02] jam: 0%| | 82/20000 [00:25<1:11:54, 4.62it/s, lr: 7.0e-05 loss: 7.817e-02] jam: 0%| | 82/20000 [00:25<1:11:54, 4.62it/s, lr: 7.0e-05 loss: 7.252e-02] jam: 0%| | 83/20000 [00:25<1:41:46, 3.26it/s, lr: 7.0e-05 loss: 7.252e-02] jam: 0%| | 83/20000 [00:26<1:41:46, 3.26it/s, lr: 7.0e-05 loss: 9.680e-02] jam: 0%| | 84/20000 [00:26<1:25:56, 3.86it/s, lr: 7.0e-05 loss: 9.680e-02] jam: 0%| | 84/20000 [00:26<1:25:56, 3.86it/s, lr: 7.0e-05 loss: 1.657e-01] jam: 0%| | 85/20000 [00:26<1:46:57, 3.10it/s, lr: 7.0e-05 loss: 1.657e-01] jam: 0%| | 85/20000 [00:26<1:46:57, 3.10it/s, lr: 7.0e-05 loss: 1.077e-01] jam: 0%| | 86/20000 [00:26<1:26:29, 3.84it/s, lr: 7.0e-05 loss: 1.077e-01] jam: 0%| | 86/20000 [00:26<1:26:29, 3.84it/s, lr: 7.0e-05 loss: 6.688e-02] jam: 0%| | 87/20000 [00:26<1:45:58, 3.13it/s, lr: 7.0e-05 loss: 6.688e-02] jam: 0%| | 87/20000 [00:27<1:45:58, 3.13it/s, lr: 7.0e-05 loss: 2.541e-01] jam: 0%| | 88/20000 [00:27<1:50:43, 3.00it/s, lr: 7.0e-05 loss: 2.541e-01] jam: 0%| | 88/20000 [00:27<1:50:43, 3.00it/s, lr: 7.0e-05 loss: 1.246e-01] jam: 0%| | 89/20000 [00:27<1:26:22, 3.84it/s, lr: 7.0e-05 loss: 1.246e-01] jam: 0%| | 89/20000 [00:27<1:26:22, 3.84it/s, lr: 7.0e-05 loss: 1.567e-01] jam: 0%| | 90/20000 [00:27<1:47:54, 3.08it/s, lr: 7.0e-05 loss: 1.567e-01] jam: 0%| | 90/20000 [00:28<1:47:54, 3.08it/s, lr: 7.0e-05 loss: 1.348e-01] jam: 0%| | 91/20000 [00:28<1:55:45, 2.87it/s, lr: 7.0e-05 loss: 1.348e-01] jam: 0%| | 91/20000 [00:28<1:55:45, 2.87it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 0%| | 92/20000 [00:28<1:53:50, 2.91it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 0%| | 92/20000 [00:28<1:53:50, 2.91it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 0%| | 93/20000 [00:28<2:00:35, 2.75it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 0%| | 93/20000 [00:29<2:00:35, 2.75it/s, lr: 7.0e-05 loss: 9.522e-02] jam: 0%| | 94/20000 [00:29<1:21:22, 4.08it/s, lr: 7.0e-05 loss: 9.522e-02] jam: 0%| | 94/20000 [00:29<1:21:22, 4.08it/s, lr: 7.0e-05 loss: 8.249e-02] jam: 0%| | 95/20000 [00:29<1:01:57, 5.35it/s, lr: 7.0e-05 loss: 8.249e-02] jam: 0%| | 95/20000 [00:29<1:01:57, 5.35it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 0%| | 96/20000 [00:29<1:06:08, 5.01it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 0%| | 96/20000 [00:29<1:06:08, 5.01it/s, lr: 7.0e-05 loss: 1.096e-01] jam: 0%| | 97/20000 [00:29<1:42:45, 3.23it/s, lr: 7.0e-05 loss: 1.096e-01] jam: 0%| | 97/20000 [00:30<1:42:45, 3.23it/s, lr: 7.0e-05 loss: 1.130e-01] jam: 0%| | 98/20000 [00:30<1:49:45, 3.02it/s, lr: 7.0e-05 loss: 1.130e-01] jam: 0%| | 98/20000 [00:30<1:49:45, 3.02it/s, lr: 7.0e-05 loss: 1.081e-01] jam: 0%| | 99/20000 [00:30<1:22:14, 4.03it/s, lr: 7.0e-05 loss: 1.081e-01] jam: 0%| | 99/20000 [00:30<1:22:14, 4.03it/s, lr: 7.0e-05 loss: 6.921e-02] jam: 0%| | 100/20000 [00:30<1:15:42, 4.38it/s, lr: 7.0e-05 loss: 6.921e-02] jam: 0%| | 100/20000 [00:31<1:15:42, 4.38it/s, lr: 7.0e-05 loss: 1.868e-01] | |
| Saving at step 100 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000100.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 101/20000 [00:31<1:15:42, 4.38it/s, lr: 7.0e-05 loss: 1.004e-01] jam: 1%| | 102/20000 [00:31<1:25:01, 3.90it/s, lr: 7.0e-05 loss: 1.004e-01] jam: 1%| | 102/20000 [00:32<1:25:01, 3.90it/s, lr: 7.0e-05 loss: 3.047e-01] jam: 1%| | 103/20000 [00:32<1:46:28, 3.11it/s, lr: 7.0e-05 loss: 3.047e-01] jam: 1%| | 103/20000 [00:32<1:46:28, 3.11it/s, lr: 7.0e-05 loss: 1.510e-01] jam: 1%| | 104/20000 [00:32<1:50:29, 3.00it/s, lr: 7.0e-05 loss: 1.510e-01] jam: 1%| | 104/20000 [00:32<1:50:29, 3.00it/s, lr: 7.0e-05 loss: 2.667e-01] jam: 1%| | 105/20000 [00:32<1:55:27, 2.87it/s, lr: 7.0e-05 loss: 2.667e-01] jam: 1%| | 105/20000 [00:33<1:55:27, 2.87it/s, lr: 7.0e-05 loss: 1.937e-01] jam: 1%| | 106/20000 [00:33<1:53:33, 2.92it/s, lr: 7.0e-05 loss: 1.937e-01] jam: 1%| | 106/20000 [00:33<1:53:33, 2.92it/s, lr: 7.0e-05 loss: 1.603e-01] jam: 1%| | 107/20000 [00:33<1:58:34, 2.80it/s, lr: 7.0e-05 loss: 1.603e-01] jam: 1%| | 107/20000 [00:33<1:58:34, 2.80it/s, lr: 7.0e-05 loss: 9.512e-02] jam: 1%| | 108/20000 [00:33<1:21:02, 4.09it/s, lr: 7.0e-05 loss: 9.512e-02] jam: 1%| | 108/20000 [00:33<1:21:02, 4.09it/s, lr: 7.0e-05 loss: 1.671e-01] jam: 1%| | 109/20000 [00:33<1:07:51, 4.89it/s, lr: 7.0e-05 loss: 1.671e-01] jam: 1%| | 109/20000 [00:34<1:07:51, 4.89it/s, lr: 7.0e-05 loss: 2.502e-01] jam: 1%| | 110/20000 [00:34<1:39:48, 3.32it/s, lr: 7.0e-05 loss: 2.502e-01] jam: 1%| | 110/20000 [00:34<1:39:48, 3.32it/s, lr: 7.0e-05 loss: 1.883e-01] jam: 1%| | 111/20000 [00:34<2:00:55, 2.74it/s, lr: 7.0e-05 loss: 1.883e-01] jam: 1%| | 111/20000 [00:34<2:00:55, 2.74it/s, lr: 7.0e-05 loss: 2.270e-01] jam: 1%| | 112/20000 [00:34<1:15:31, 4.39it/s, lr: 7.0e-05 loss: 2.270e-01] jam: 1%| | 112/20000 [00:34<1:15:31, 4.39it/s, lr: 7.0e-05 loss: 1.507e-01] jam: 1%| | 113/20000 [00:34<1:18:34, 4.22it/s, lr: 7.0e-05 loss: 1.507e-01] jam: 1%| | 113/20000 [00:35<1:18:34, 4.22it/s, lr: 7.0e-05 loss: 1.380e-01] jam: 1%| | 114/20000 [00:35<1:49:26, 3.03it/s, lr: 7.0e-05 loss: 1.380e-01] jam: 1%| | 114/20000 [00:35<1:49:26, 3.03it/s, lr: 7.0e-05 loss: 1.211e-01] jam: 1%| | 115/20000 [00:35<1:52:01, 2.96it/s, lr: 7.0e-05 loss: 1.211e-01] jam: 1%| | 115/20000 [00:36<1:52:01, 2.96it/s, lr: 7.0e-05 loss: 2.026e-01] jam: 1%| | 116/20000 [00:36<1:57:42, 2.82it/s, lr: 7.0e-05 loss: 2.026e-01] jam: 1%| | 116/20000 [00:36<1:57:42, 2.82it/s, lr: 7.0e-05 loss: 1.438e-01] jam: 1%| | 117/20000 [00:36<1:17:27, 4.28it/s, lr: 7.0e-05 loss: 1.438e-01] jam: 1%| | 117/20000 [00:36<1:17:27, 4.28it/s, lr: 7.0e-05 loss: 1.116e-01] jam: 1%| | 118/20000 [00:36<1:42:17, 3.24it/s, lr: 7.0e-05 loss: 1.116e-01] jam: 1%| | 118/20000 [00:36<1:42:17, 3.24it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 1%| | 119/20000 [00:36<1:25:15, 3.89it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 1%| | 119/20000 [00:37<1:25:15, 3.89it/s, lr: 7.0e-05 loss: 1.768e-01] jam: 1%| | 120/20000 [00:37<1:52:07, 2.95it/s, lr: 7.0e-05 loss: 1.768e-01] jam: 1%| | 120/20000 [00:37<1:52:07, 2.95it/s, lr: 7.0e-05 loss: 1.970e-01] jam: 1%| | 121/20000 [00:37<1:59:10, 2.78it/s, lr: 7.0e-05 loss: 1.970e-01] jam: 1%| | 121/20000 [00:37<1:59:10, 2.78it/s, lr: 7.0e-05 loss: 1.769e-01] jam: 1%| | 122/20000 [00:37<1:18:03, 4.24it/s, lr: 7.0e-05 loss: 1.769e-01] jam: 1%| | 122/20000 [00:38<1:18:03, 4.24it/s, lr: 7.0e-05 loss: 1.308e-01] jam: 1%| | 123/20000 [00:38<1:41:26, 3.27it/s, lr: 7.0e-05 loss: 1.308e-01] jam: 1%| | 123/20000 [00:38<1:41:26, 3.27it/s, lr: 7.0e-05 loss: 1.212e-01] jam: 1%| | 124/20000 [00:38<1:49:11, 3.03it/s, lr: 7.0e-05 loss: 1.212e-01] jam: 1%| | 124/20000 [00:38<1:49:11, 3.03it/s, lr: 7.0e-05 loss: 4.282e-02] jam: 1%| | 125/20000 [00:38<1:55:54, 2.86it/s, lr: 7.0e-05 loss: 4.282e-02] jam: 1%| | 125/20000 [00:39<1:55:54, 2.86it/s, lr: 7.0e-05 loss: 1.379e-01] | |
| Saving at step 125 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000125.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 126/20000 [00:39<1:55:53, 2.86it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 1%| | 127/20000 [00:39<1:42:11, 3.24it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 1%| | 127/20000 [00:40<1:42:11, 3.24it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 1%| | 128/20000 [00:40<1:51:33, 2.97it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 1%| | 128/20000 [00:40<1:51:33, 2.97it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 1%| | 129/20000 [00:40<1:52:04, 2.95it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 1%| | 129/20000 [00:40<1:52:04, 2.95it/s, lr: 7.0e-05 loss: 2.639e-01] jam: 1%| | 130/20000 [00:40<1:28:21, 3.75it/s, lr: 7.0e-05 loss: 2.639e-01] jam: 1%| | 130/20000 [00:41<1:28:21, 3.75it/s, lr: 7.0e-05 loss: 1.892e-01] jam: 1%| | 131/20000 [00:41<1:47:03, 3.09it/s, lr: 7.0e-05 loss: 1.892e-01] jam: 1%| | 131/20000 [00:41<1:47:03, 3.09it/s, lr: 7.0e-05 loss: 5.020e-02] jam: 1%| | 132/20000 [00:41<1:42:30, 3.23it/s, lr: 7.0e-05 loss: 5.020e-02] jam: 1%| | 132/20000 [00:41<1:42:30, 3.23it/s, lr: 7.0e-05 loss: 1.662e-01] jam: 1%| | 133/20000 [00:41<1:17:42, 4.26it/s, lr: 7.0e-05 loss: 1.662e-01] jam: 1%| | 133/20000 [00:41<1:17:42, 4.26it/s, lr: 7.0e-05 loss: 2.348e-01] jam: 1%| | 134/20000 [00:41<1:18:39, 4.21it/s, lr: 7.0e-05 loss: 2.348e-01] jam: 1%| | 134/20000 [00:42<1:18:39, 4.21it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 1%| | 135/20000 [00:42<1:43:49, 3.19it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 1%| | 135/20000 [00:42<1:43:49, 3.19it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 1%| | 136/20000 [00:42<1:57:20, 2.82it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 1%| | 136/20000 [00:42<1:57:20, 2.82it/s, lr: 7.0e-05 loss: 1.570e-01] jam: 1%| | 137/20000 [00:42<1:19:27, 4.17it/s, lr: 7.0e-05 loss: 1.570e-01] jam: 1%| | 137/20000 [00:43<1:19:27, 4.17it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 1%| | 138/20000 [00:43<1:11:29, 4.63it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 1%| | 138/20000 [00:43<1:11:29, 4.63it/s, lr: 7.0e-05 loss: 1.105e-01] jam: 1%| | 139/20000 [00:43<1:41:50, 3.25it/s, lr: 7.0e-05 loss: 1.105e-01] jam: 1%| | 139/20000 [00:43<1:41:50, 3.25it/s, lr: 7.0e-05 loss: 7.767e-02] jam: 1%| | 140/20000 [00:43<1:48:55, 3.04it/s, lr: 7.0e-05 loss: 7.767e-02] jam: 1%| | 140/20000 [00:43<1:48:55, 3.04it/s, lr: 7.0e-05 loss: 1.461e-01] jam: 1%| | 141/20000 [00:43<1:31:05, 3.63it/s, lr: 7.0e-05 loss: 1.461e-01] jam: 1%| | 141/20000 [00:44<1:31:05, 3.63it/s, lr: 7.0e-05 loss: 9.132e-02] jam: 1%| | 142/20000 [00:44<1:53:18, 2.92it/s, lr: 7.0e-05 loss: 9.132e-02] jam: 1%| | 142/20000 [00:44<1:53:18, 2.92it/s, lr: 7.0e-05 loss: 1.241e-01] jam: 1%| | 143/20000 [00:44<1:47:36, 3.08it/s, lr: 7.0e-05 loss: 1.241e-01] jam: 1%| | 143/20000 [00:44<1:47:36, 3.08it/s, lr: 7.0e-05 loss: 5.918e-02] jam: 1%| | 144/20000 [00:44<1:13:06, 4.53it/s, lr: 7.0e-05 loss: 5.918e-02] jam: 1%| | 144/20000 [00:45<1:13:06, 4.53it/s, lr: 7.0e-05 loss: 1.220e-01] jam: 1%| | 145/20000 [00:45<1:39:03, 3.34it/s, lr: 7.0e-05 loss: 1.220e-01] jam: 1%| | 145/20000 [00:45<1:39:03, 3.34it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 1%| | 146/20000 [00:45<1:25:01, 3.89it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 1%| | 146/20000 [00:45<1:25:01, 3.89it/s, lr: 7.0e-05 loss: 1.942e-01] jam: 1%| | 147/20000 [00:45<1:13:40, 4.49it/s, lr: 7.0e-05 loss: 1.942e-01] jam: 1%| | 147/20000 [00:45<1:13:40, 4.49it/s, lr: 7.0e-05 loss: 1.557e-01] jam: 1%| | 148/20000 [00:45<1:19:49, 4.15it/s, lr: 7.0e-05 loss: 1.557e-01] jam: 1%| | 148/20000 [00:46<1:19:49, 4.15it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 1%| | 149/20000 [00:46<1:06:14, 4.99it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 1%| | 149/20000 [00:46<1:06:14, 4.99it/s, lr: 7.0e-05 loss: 1.795e-01] jam: 1%| | 150/20000 [00:46<1:04:57, 5.09it/s, lr: 7.0e-05 loss: 1.795e-01] jam: 1%| | 150/20000 [00:46<1:04:57, 5.09it/s, lr: 7.0e-05 loss: 1.321e-01] | |
| Saving at step 150 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000150.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 151/20000 [00:47<1:04:56, 5.09it/s, lr: 7.0e-05 loss: 8.730e-02] jam: 1%| | 152/20000 [00:47<1:10:16, 4.71it/s, lr: 7.0e-05 loss: 8.730e-02] jam: 1%| | 152/20000 [00:47<1:10:16, 4.71it/s, lr: 7.0e-05 loss: 3.190e-02] jam: 1%| | 153/20000 [00:47<1:00:57, 5.43it/s, lr: 7.0e-05 loss: 3.190e-02] jam: 1%| | 153/20000 [00:47<1:00:57, 5.43it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 1%| | 154/20000 [00:47<1:12:14, 4.58it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 1%| | 154/20000 [00:47<1:12:14, 4.58it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 1%| | 155/20000 [00:47<1:05:45, 5.03it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 1%| | 155/20000 [00:48<1:05:45, 5.03it/s, lr: 7.0e-05 loss: 1.532e-01] jam: 1%| | 156/20000 [00:48<1:11:45, 4.61it/s, lr: 7.0e-05 loss: 1.532e-01] jam: 1%| | 156/20000 [00:48<1:11:45, 4.61it/s, lr: 7.0e-05 loss: 8.509e-02] jam: 1%| | 157/20000 [00:48<1:35:23, 3.47it/s, lr: 7.0e-05 loss: 8.509e-02] jam: 1%| | 157/20000 [00:48<1:35:23, 3.47it/s, lr: 7.0e-05 loss: 1.682e-01] jam: 1%| | 158/20000 [00:48<1:06:45, 4.95it/s, lr: 7.0e-05 loss: 1.682e-01] jam: 1%| | 158/20000 [00:48<1:06:45, 4.95it/s, lr: 7.0e-05 loss: 1.227e-01] jam: 1%| | 159/20000 [00:48<1:10:16, 4.71it/s, lr: 7.0e-05 loss: 1.227e-01] jam: 1%| | 159/20000 [00:49<1:10:16, 4.71it/s, lr: 7.0e-05 loss: 1.638e-01] jam: 1%| | 160/20000 [00:49<1:38:05, 3.37it/s, lr: 7.0e-05 loss: 1.638e-01] jam: 1%| | 160/20000 [00:49<1:38:05, 3.37it/s, lr: 7.0e-05 loss: 3.046e-01] jam: 1%| | 161/20000 [00:49<1:17:30, 4.27it/s, lr: 7.0e-05 loss: 3.046e-01] jam: 1%| | 161/20000 [00:49<1:17:30, 4.27it/s, lr: 7.0e-05 loss: 1.643e-01] jam: 1%| | 162/20000 [00:49<1:16:20, 4.33it/s, lr: 7.0e-05 loss: 1.643e-01] jam: 1%| | 162/20000 [00:49<1:16:20, 4.33it/s, lr: 7.0e-05 loss: 3.747e-02] jam: 1%| | 163/20000 [00:49<1:43:02, 3.21it/s, lr: 7.0e-05 loss: 3.747e-02] jam: 1%| | 163/20000 [00:50<1:43:02, 3.21it/s, lr: 7.0e-05 loss: 2.345e-01] jam: 1%| | 164/20000 [00:50<1:09:24, 4.76it/s, lr: 7.0e-05 loss: 2.345e-01] jam: 1%| | 164/20000 [00:50<1:09:24, 4.76it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 1%| | 165/20000 [00:50<1:39:31, 3.32it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 1%| | 165/20000 [00:50<1:39:31, 3.32it/s, lr: 7.0e-05 loss: 4.579e-02] jam: 1%| | 166/20000 [00:50<1:31:19, 3.62it/s, lr: 7.0e-05 loss: 4.579e-02] jam: 1%| | 166/20000 [00:50<1:31:19, 3.62it/s, lr: 7.0e-05 loss: 2.188e-01] jam: 1%| | 167/20000 [00:50<1:11:07, 4.65it/s, lr: 7.0e-05 loss: 2.188e-01] jam: 1%| | 167/20000 [00:51<1:11:07, 4.65it/s, lr: 7.0e-05 loss: 2.532e-01] jam: 1%| | 168/20000 [00:51<1:08:35, 4.82it/s, lr: 7.0e-05 loss: 2.532e-01] jam: 1%| | 168/20000 [00:51<1:08:35, 4.82it/s, lr: 7.0e-05 loss: 1.039e-01] jam: 1%| | 169/20000 [00:51<1:39:00, 3.34it/s, lr: 7.0e-05 loss: 1.039e-01] jam: 1%| | 169/20000 [00:51<1:39:00, 3.34it/s, lr: 7.0e-05 loss: 6.386e-02] jam: 1%| | 170/20000 [00:51<1:56:53, 2.83it/s, lr: 7.0e-05 loss: 6.386e-02] jam: 1%| | 170/20000 [00:51<1:56:53, 2.83it/s, lr: 7.0e-05 loss: 3.455e-01] jam: 1%| | 171/20000 [00:51<1:17:26, 4.27it/s, lr: 7.0e-05 loss: 3.455e-01] jam: 1%| | 171/20000 [00:52<1:17:26, 4.27it/s, lr: 7.0e-05 loss: 1.934e-01] jam: 1%| | 172/20000 [00:52<1:04:50, 5.10it/s, lr: 7.0e-05 loss: 1.934e-01] jam: 1%| | 172/20000 [00:52<1:04:50, 5.10it/s, lr: 7.0e-05 loss: 1.165e-01] jam: 1%| | 173/20000 [00:52<1:08:25, 4.83it/s, lr: 7.0e-05 loss: 1.165e-01] jam: 1%| | 173/20000 [00:52<1:08:25, 4.83it/s, lr: 7.0e-05 loss: 2.212e-01] jam: 1%| | 174/20000 [00:52<1:41:17, 3.26it/s, lr: 7.0e-05 loss: 2.212e-01] jam: 1%| | 174/20000 [00:52<1:41:17, 3.26it/s, lr: 7.0e-05 loss: 1.375e-01] jam: 1%| | 175/20000 [00:52<1:20:06, 4.12it/s, lr: 7.0e-05 loss: 1.375e-01] jam: 1%| | 175/20000 [00:53<1:20:06, 4.12it/s, lr: 7.0e-05 loss: 1.224e-01] | |
| Saving at step 175 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000175.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 176/20000 [00:53<1:20:06, 4.12it/s, lr: 7.0e-05 loss: 1.155e-01] jam: 1%| | 177/20000 [00:53<1:15:27, 4.38it/s, lr: 7.0e-05 loss: 1.155e-01] jam: 1%| | 177/20000 [00:54<1:15:27, 4.38it/s, lr: 7.0e-05 loss: 1.905e-01] jam: 1%| | 178/20000 [00:54<1:12:12, 4.58it/s, lr: 7.0e-05 loss: 1.905e-01] jam: 1%| | 178/20000 [00:54<1:12:12, 4.58it/s, lr: 7.0e-05 loss: 5.020e-02] jam: 1%| | 179/20000 [00:54<1:39:54, 3.31it/s, lr: 7.0e-05 loss: 5.020e-02] jam: 1%| | 179/20000 [00:54<1:39:54, 3.31it/s, lr: 7.0e-05 loss: 1.952e-01] jam: 1%| | 180/20000 [00:54<1:22:30, 4.00it/s, lr: 7.0e-05 loss: 1.952e-01] jam: 1%| | 180/20000 [00:54<1:22:30, 4.00it/s, lr: 7.0e-05 loss: 1.348e-01] jam: 1%| | 181/20000 [00:54<1:15:45, 4.36it/s, lr: 7.0e-05 loss: 1.348e-01] jam: 1%| | 181/20000 [00:55<1:15:45, 4.36it/s, lr: 7.0e-05 loss: 1.642e-01] jam: 1%| | 182/20000 [00:55<1:45:28, 3.13it/s, lr: 7.0e-05 loss: 1.642e-01] jam: 1%| | 182/20000 [00:55<1:45:28, 3.13it/s, lr: 7.0e-05 loss: 1.970e-01] jam: 1%| | 183/20000 [00:55<1:59:07, 2.77it/s, lr: 7.0e-05 loss: 1.970e-01] jam: 1%| | 183/20000 [00:55<1:59:07, 2.77it/s, lr: 7.0e-05 loss: 1.502e-01] jam: 1%| | 184/20000 [00:55<1:18:58, 4.18it/s, lr: 7.0e-05 loss: 1.502e-01] jam: 1%| | 184/20000 [00:55<1:18:58, 4.18it/s, lr: 7.0e-05 loss: 5.981e-02] jam: 1%| | 185/20000 [00:55<1:06:51, 4.94it/s, lr: 7.0e-05 loss: 5.981e-02] jam: 1%| | 185/20000 [00:56<1:06:51, 4.94it/s, lr: 7.0e-05 loss: 9.786e-02] jam: 1%| | 186/20000 [00:56<1:38:08, 3.36it/s, lr: 7.0e-05 loss: 9.786e-02] jam: 1%| | 186/20000 [00:56<1:38:08, 3.36it/s, lr: 7.0e-05 loss: 1.713e-01] jam: 1%| | 187/20000 [00:56<1:48:00, 3.06it/s, lr: 7.0e-05 loss: 1.713e-01] jam: 1%| | 187/20000 [00:56<1:48:00, 3.06it/s, lr: 7.0e-05 loss: 1.383e-01] jam: 1%| | 188/20000 [00:56<1:26:40, 3.81it/s, lr: 7.0e-05 loss: 1.383e-01] jam: 1%| | 188/20000 [00:57<1:26:40, 3.81it/s, lr: 7.0e-05 loss: 9.283e-02] jam: 1%| | 189/20000 [00:57<1:44:14, 3.17it/s, lr: 7.0e-05 loss: 9.283e-02] jam: 1%| | 189/20000 [00:57<1:44:14, 3.17it/s, lr: 7.0e-05 loss: 1.887e-02] jam: 1%| | 190/20000 [00:57<1:21:58, 4.03it/s, lr: 7.0e-05 loss: 1.887e-02] jam: 1%| | 190/20000 [00:57<1:21:58, 4.03it/s, lr: 7.0e-05 loss: 6.336e-02] jam: 1%| | 191/20000 [00:57<1:48:01, 3.06it/s, lr: 7.0e-05 loss: 6.336e-02] jam: 1%| | 191/20000 [00:58<1:48:01, 3.06it/s, lr: 7.0e-05 loss: 1.838e-01] jam: 1%| | 192/20000 [00:58<1:50:49, 2.98it/s, lr: 7.0e-05 loss: 1.838e-01] jam: 1%| | 192/20000 [00:58<1:50:49, 2.98it/s, lr: 7.0e-05 loss: 8.802e-02] jam: 1%| | 193/20000 [00:58<1:57:38, 2.81it/s, lr: 7.0e-05 loss: 8.802e-02] jam: 1%| | 193/20000 [00:58<1:57:38, 2.81it/s, lr: 7.0e-05 loss: 1.511e-01] jam: 1%| | 194/20000 [00:58<1:54:00, 2.90it/s, lr: 7.0e-05 loss: 1.511e-01] jam: 1%| | 194/20000 [00:59<1:54:00, 2.90it/s, lr: 7.0e-05 loss: 1.783e-01] jam: 1%| | 195/20000 [00:59<1:28:05, 3.75it/s, lr: 7.0e-05 loss: 1.783e-01] jam: 1%| | 195/20000 [00:59<1:28:05, 3.75it/s, lr: 7.0e-05 loss: 2.139e-01] jam: 1%| | 196/20000 [00:59<1:05:05, 5.07it/s, lr: 7.0e-05 loss: 2.139e-01] jam: 1%| | 196/20000 [00:59<1:05:05, 5.07it/s, lr: 7.0e-05 loss: 1.110e-01] jam: 1%| | 197/20000 [00:59<1:22:03, 4.02it/s, lr: 7.0e-05 loss: 1.110e-01] jam: 1%| | 197/20000 [00:59<1:22:03, 4.02it/s, lr: 7.0e-05 loss: 1.497e-01] jam: 1%| | 198/20000 [00:59<1:14:56, 4.40it/s, lr: 7.0e-05 loss: 1.497e-01] jam: 1%| | 198/20000 [00:59<1:14:56, 4.40it/s, lr: 7.0e-05 loss: 7.118e-02] jam: 1%| | 199/20000 [00:59<1:12:09, 4.57it/s, lr: 7.0e-05 loss: 7.118e-02] jam: 1%| | 199/20000 [01:00<1:12:09, 4.57it/s, lr: 7.0e-05 loss: 2.310e-01] jam: 1%| | 200/20000 [01:00<1:21:29, 4.05it/s, lr: 7.0e-05 loss: 2.310e-01] jam: 1%| | 200/20000 [01:00<1:21:29, 4.05it/s, lr: 7.0e-05 loss: 1.289e-01] | |
| Saving at step 200 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000200.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 201/20000 [01:01<1:21:28, 4.05it/s, lr: 7.0e-05 loss: 2.704e-01] jam: 1%| | 202/20000 [01:01<1:13:12, 4.51it/s, lr: 7.0e-05 loss: 2.704e-01] jam: 1%| | 202/20000 [01:01<1:13:12, 4.51it/s, lr: 7.0e-05 loss: 4.663e-02] jam: 1%| | 203/20000 [01:01<1:35:39, 3.45it/s, lr: 7.0e-05 loss: 4.663e-02] jam: 1%| | 203/20000 [01:01<1:35:39, 3.45it/s, lr: 7.0e-05 loss: 9.745e-02] jam: 1%| | 204/20000 [01:01<1:53:12, 2.91it/s, lr: 7.0e-05 loss: 9.745e-02] jam: 1%| | 204/20000 [01:02<1:53:12, 2.91it/s, lr: 7.0e-05 loss: 1.535e-01] jam: 1%| | 205/20000 [01:02<1:26:21, 3.82it/s, lr: 7.0e-05 loss: 1.535e-01] jam: 1%| | 205/20000 [01:02<1:26:21, 3.82it/s, lr: 7.0e-05 loss: 1.076e-01] jam: 1%| | 206/20000 [01:02<1:16:07, 4.33it/s, lr: 7.0e-05 loss: 1.076e-01] jam: 1%| | 206/20000 [01:02<1:16:07, 4.33it/s, lr: 7.0e-05 loss: 7.510e-02] jam: 1%| | 207/20000 [01:02<1:45:08, 3.14it/s, lr: 7.0e-05 loss: 7.510e-02] jam: 1%| | 207/20000 [01:03<1:45:08, 3.14it/s, lr: 7.0e-05 loss: 2.068e-01] jam: 1%| | 208/20000 [01:03<1:43:56, 3.17it/s, lr: 7.0e-05 loss: 2.068e-01] jam: 1%| | 208/20000 [01:03<1:43:56, 3.17it/s, lr: 7.0e-05 loss: 7.001e-02] jam: 1%| | 209/20000 [01:03<1:49:27, 3.01it/s, lr: 7.0e-05 loss: 7.001e-02] jam: 1%| | 209/20000 [01:03<1:49:27, 3.01it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 1%| | 210/20000 [01:03<1:35:29, 3.45it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 1%| | 210/20000 [01:03<1:35:29, 3.45it/s, lr: 7.0e-05 loss: 2.069e-01] jam: 1%| | 211/20000 [01:03<1:06:39, 4.95it/s, lr: 7.0e-05 loss: 2.069e-01] jam: 1%| | 211/20000 [01:04<1:06:39, 4.95it/s, lr: 7.0e-05 loss: 1.054e-01] jam: 1%| | 212/20000 [01:04<1:14:21, 4.44it/s, lr: 7.0e-05 loss: 1.054e-01] jam: 1%| | 212/20000 [01:04<1:14:21, 4.44it/s, lr: 7.0e-05 loss: 1.855e-01] jam: 1%| | 213/20000 [01:04<1:39:50, 3.30it/s, lr: 7.0e-05 loss: 1.855e-01] jam: 1%| | 213/20000 [01:04<1:39:50, 3.30it/s, lr: 7.0e-05 loss: 1.017e-01] jam: 1%| | 214/20000 [01:04<1:48:05, 3.05it/s, lr: 7.0e-05 loss: 1.017e-01] jam: 1%| | 214/20000 [01:04<1:48:05, 3.05it/s, lr: 7.0e-05 loss: 2.668e-01] jam: 1%| | 215/20000 [01:04<1:15:10, 4.39it/s, lr: 7.0e-05 loss: 2.668e-01] jam: 1%| | 215/20000 [01:05<1:15:10, 4.39it/s, lr: 7.0e-05 loss: 2.455e-01] jam: 1%| | 216/20000 [01:05<1:32:29, 3.57it/s, lr: 7.0e-05 loss: 2.455e-01] jam: 1%| | 216/20000 [01:05<1:32:29, 3.57it/s, lr: 7.0e-05 loss: 1.436e-01] jam: 1%| | 217/20000 [01:05<1:25:14, 3.87it/s, lr: 7.0e-05 loss: 1.436e-01] jam: 1%| | 217/20000 [01:05<1:25:14, 3.87it/s, lr: 7.0e-05 loss: 1.020e-01] jam: 1%| | 218/20000 [01:05<1:13:48, 4.47it/s, lr: 7.0e-05 loss: 1.020e-01] jam: 1%| | 218/20000 [01:05<1:13:48, 4.47it/s, lr: 7.0e-05 loss: 3.042e-01] jam: 1%| | 219/20000 [01:05<1:10:21, 4.69it/s, lr: 7.0e-05 loss: 3.042e-01] jam: 1%| | 219/20000 [01:06<1:10:21, 4.69it/s, lr: 7.0e-05 loss: 9.574e-02] jam: 1%| | 220/20000 [01:06<1:13:01, 4.51it/s, lr: 7.0e-05 loss: 9.574e-02] jam: 1%| | 220/20000 [01:06<1:13:01, 4.51it/s, lr: 7.0e-05 loss: 1.354e-01] jam: 1%| | 221/20000 [01:06<1:42:13, 3.22it/s, lr: 7.0e-05 loss: 1.354e-01] jam: 1%| | 221/20000 [01:06<1:42:13, 3.22it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 1%| | 222/20000 [01:06<1:25:45, 3.84it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 1%| | 222/20000 [01:06<1:25:45, 3.84it/s, lr: 7.0e-05 loss: 6.661e-02] jam: 1%| | 223/20000 [01:06<1:12:47, 4.53it/s, lr: 7.0e-05 loss: 6.661e-02] jam: 1%| | 223/20000 [01:07<1:12:47, 4.53it/s, lr: 7.0e-05 loss: 9.149e-02] jam: 1%| | 224/20000 [01:07<1:44:08, 3.17it/s, lr: 7.0e-05 loss: 9.149e-02] jam: 1%| | 224/20000 [01:07<1:44:08, 3.17it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 1%| | 225/20000 [01:07<1:57:32, 2.80it/s, lr: 7.0e-05 loss: 1.758e-01] jam: 1%| | 225/20000 [01:08<1:57:32, 2.80it/s, lr: 7.0e-05 loss: 1.055e-01] | |
| Saving at step 225 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000225.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%| | 226/20000 [01:08<1:57:32, 2.80it/s, lr: 7.0e-05 loss: 1.025e-01] jam: 1%| | 227/20000 [01:08<1:41:56, 3.23it/s, lr: 7.0e-05 loss: 1.025e-01] jam: 1%| | 227/20000 [01:09<1:41:56, 3.23it/s, lr: 7.0e-05 loss: 1.080e-01] jam: 1%| | 228/20000 [01:09<1:52:08, 2.94it/s, lr: 7.0e-05 loss: 1.080e-01] jam: 1%| | 228/20000 [01:09<1:52:08, 2.94it/s, lr: 7.0e-05 loss: 3.731e-01] jam: 1%| | 229/20000 [01:09<1:15:55, 4.34it/s, lr: 7.0e-05 loss: 3.731e-01] jam: 1%| | 229/20000 [01:09<1:15:55, 4.34it/s, lr: 7.0e-05 loss: 8.675e-02] jam: 1%| | 230/20000 [01:09<1:41:45, 3.24it/s, lr: 7.0e-05 loss: 8.675e-02] jam: 1%| | 230/20000 [01:09<1:41:45, 3.24it/s, lr: 7.0e-05 loss: 1.559e-01] jam: 1%| | 231/20000 [01:09<1:19:26, 4.15it/s, lr: 7.0e-05 loss: 1.559e-01] jam: 1%| | 231/20000 [01:10<1:19:26, 4.15it/s, lr: 7.0e-05 loss: 2.365e-01] jam: 1%| | 232/20000 [01:10<1:46:52, 3.08it/s, lr: 7.0e-05 loss: 2.365e-01] jam: 1%| | 232/20000 [01:10<1:46:52, 3.08it/s, lr: 7.0e-05 loss: 9.616e-02] jam: 1%| | 233/20000 [01:10<1:50:02, 2.99it/s, lr: 7.0e-05 loss: 9.616e-02] jam: 1%| | 233/20000 [01:11<1:50:02, 2.99it/s, lr: 7.0e-05 loss: 1.237e-01] jam: 1%| | 234/20000 [01:11<1:57:33, 2.80it/s, lr: 7.0e-05 loss: 1.237e-01] jam: 1%| | 234/20000 [01:11<1:57:33, 2.80it/s, lr: 7.0e-05 loss: 1.161e-01] jam: 1%| | 235/20000 [01:11<1:18:21, 4.20it/s, lr: 7.0e-05 loss: 1.161e-01] jam: 1%| | 235/20000 [01:11<1:18:21, 4.20it/s, lr: 7.0e-05 loss: 1.325e-01] jam: 1%| | 236/20000 [01:11<1:01:07, 5.39it/s, lr: 7.0e-05 loss: 1.325e-01] jam: 1%| | 236/20000 [01:11<1:01:07, 5.39it/s, lr: 7.0e-05 loss: 2.621e-01] jam: 1%| | 237/20000 [01:11<1:03:01, 5.23it/s, lr: 7.0e-05 loss: 2.621e-01] jam: 1%| | 237/20000 [01:11<1:03:01, 5.23it/s, lr: 7.0e-05 loss: 1.184e-01] jam: 1%| | 238/20000 [01:11<1:40:49, 3.27it/s, lr: 7.0e-05 loss: 1.184e-01] jam: 1%| | 238/20000 [01:12<1:40:49, 3.27it/s, lr: 7.0e-05 loss: 1.297e-01] jam: 1%| | 239/20000 [01:12<1:14:10, 4.44it/s, lr: 7.0e-05 loss: 1.297e-01] jam: 1%| | 239/20000 [01:12<1:14:10, 4.44it/s, lr: 7.0e-05 loss: 1.008e-01] jam: 1%| | 240/20000 [01:12<1:39:43, 3.30it/s, lr: 7.0e-05 loss: 1.008e-01] jam: 1%| | 240/20000 [01:12<1:39:43, 3.30it/s, lr: 7.0e-05 loss: 1.972e-01] jam: 1%| | 241/20000 [01:12<1:20:43, 4.08it/s, lr: 7.0e-05 loss: 1.972e-01] jam: 1%| | 241/20000 [01:12<1:20:43, 4.08it/s, lr: 7.0e-05 loss: 1.608e-01] jam: 1%| | 242/20000 [01:12<1:10:21, 4.68it/s, lr: 7.0e-05 loss: 1.608e-01] jam: 1%| | 242/20000 [01:13<1:10:21, 4.68it/s, lr: 7.0e-05 loss: 3.395e-01] jam: 1%| | 243/20000 [01:13<1:07:31, 4.88it/s, lr: 7.0e-05 loss: 3.395e-01] jam: 1%| | 243/20000 [01:13<1:07:31, 4.88it/s, lr: 7.0e-05 loss: 2.156e-01] jam: 1%| | 244/20000 [01:13<1:14:14, 4.44it/s, lr: 7.0e-05 loss: 2.156e-01] jam: 1%| | 244/20000 [01:13<1:14:14, 4.44it/s, lr: 7.0e-05 loss: 1.934e-01] jam: 1%| | 245/20000 [01:13<1:09:53, 4.71it/s, lr: 7.0e-05 loss: 1.934e-01] jam: 1%| | 245/20000 [01:13<1:09:53, 4.71it/s, lr: 7.0e-05 loss: 1.770e-01] jam: 1%| | 246/20000 [01:13<1:06:33, 4.95it/s, lr: 7.0e-05 loss: 1.770e-01] jam: 1%| | 246/20000 [01:13<1:06:33, 4.95it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 1%| | 247/20000 [01:13<1:08:19, 4.82it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 1%| | 247/20000 [01:14<1:08:19, 4.82it/s, lr: 7.0e-05 loss: 2.032e-01] jam: 1%| | 248/20000 [01:14<1:08:52, 4.78it/s, lr: 7.0e-05 loss: 2.032e-01] jam: 1%| | 248/20000 [01:14<1:08:52, 4.78it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 1%| | 249/20000 [01:14<1:41:23, 3.25it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 1%| | 249/20000 [01:14<1:41:23, 3.25it/s, lr: 7.0e-05 loss: 9.930e-02] jam: 1%|▏ | 250/20000 [01:14<1:20:10, 4.11it/s, lr: 7.0e-05 loss: 9.930e-02] jam: 1%|▏ | 250/20000 [01:15<1:20:10, 4.11it/s, lr: 7.0e-05 loss: 8.415e-02] | |
| Saving at step 250 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000250.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%|▏ | 251/20000 [01:15<1:20:10, 4.11it/s, lr: 7.0e-05 loss: 2.248e-01] jam: 1%|▏ | 252/20000 [01:15<1:35:55, 3.43it/s, lr: 7.0e-05 loss: 2.248e-01] jam: 1%|▏ | 252/20000 [01:16<1:35:55, 3.43it/s, lr: 7.0e-05 loss: 1.222e-01] jam: 1%|▏ | 253/20000 [01:16<1:19:49, 4.12it/s, lr: 7.0e-05 loss: 1.222e-01] jam: 1%|▏ | 253/20000 [01:16<1:19:49, 4.12it/s, lr: 7.0e-05 loss: 2.283e-01] jam: 1%|▏ | 254/20000 [01:16<1:11:48, 4.58it/s, lr: 7.0e-05 loss: 2.283e-01] jam: 1%|▏ | 254/20000 [01:16<1:11:48, 4.58it/s, lr: 7.0e-05 loss: 1.810e-01] jam: 1%|▏ | 255/20000 [01:16<1:05:11, 5.05it/s, lr: 7.0e-05 loss: 1.810e-01] jam: 1%|▏ | 255/20000 [01:16<1:05:11, 5.05it/s, lr: 7.0e-05 loss: 1.086e-01] jam: 1%|▏ | 256/20000 [01:16<1:01:41, 5.33it/s, lr: 7.0e-05 loss: 1.086e-01] jam: 1%|▏ | 256/20000 [01:16<1:01:41, 5.33it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 1%|▏ | 257/20000 [01:16<56:55, 5.78it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 1%|▏ | 257/20000 [01:17<56:55, 5.78it/s, lr: 7.0e-05 loss: 1.672e-01] jam: 1%|▏ | 258/20000 [01:17<1:37:02, 3.39it/s, lr: 7.0e-05 loss: 1.672e-01] jam: 1%|▏ | 258/20000 [01:17<1:37:02, 3.39it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 1%|▏ | 259/20000 [01:17<1:22:02, 4.01it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 1%|▏ | 259/20000 [01:17<1:22:02, 4.01it/s, lr: 7.0e-05 loss: 1.823e-01] jam: 1%|▏ | 260/20000 [01:17<1:10:43, 4.65it/s, lr: 7.0e-05 loss: 1.823e-01] jam: 1%|▏ | 260/20000 [01:17<1:10:43, 4.65it/s, lr: 7.0e-05 loss: 1.437e-01] jam: 1%|▏ | 261/20000 [01:17<1:08:06, 4.83it/s, lr: 7.0e-05 loss: 1.437e-01] jam: 1%|▏ | 261/20000 [01:17<1:08:06, 4.83it/s, lr: 7.0e-05 loss: 1.482e-01] jam: 1%|▏ | 262/20000 [01:17<1:10:00, 4.70it/s, lr: 7.0e-05 loss: 1.482e-01] jam: 1%|▏ | 262/20000 [01:18<1:10:00, 4.70it/s, lr: 7.0e-05 loss: 2.760e-01] jam: 1%|▏ | 263/20000 [01:18<1:10:40, 4.65it/s, lr: 7.0e-05 loss: 2.760e-01] jam: 1%|▏ | 263/20000 [01:18<1:10:40, 4.65it/s, lr: 7.0e-05 loss: 7.980e-02] jam: 1%|▏ | 264/20000 [01:18<1:15:04, 4.38it/s, lr: 7.0e-05 loss: 7.980e-02] jam: 1%|▏ | 264/20000 [01:18<1:15:04, 4.38it/s, lr: 7.0e-05 loss: 7.222e-02] jam: 1%|▏ | 265/20000 [01:18<1:12:55, 4.51it/s, lr: 7.0e-05 loss: 7.222e-02] jam: 1%|▏ | 265/20000 [01:18<1:12:55, 4.51it/s, lr: 7.0e-05 loss: 2.454e-01] jam: 1%|▏ | 266/20000 [01:18<1:14:59, 4.39it/s, lr: 7.0e-05 loss: 2.454e-01] jam: 1%|▏ | 266/20000 [01:19<1:14:59, 4.39it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 1%|▏ | 267/20000 [01:19<1:08:29, 4.80it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 1%|▏ | 267/20000 [01:19<1:08:29, 4.80it/s, lr: 7.0e-05 loss: 1.983e-01] jam: 1%|▏ | 268/20000 [01:19<1:06:15, 4.96it/s, lr: 7.0e-05 loss: 1.983e-01] jam: 1%|▏ | 268/20000 [01:19<1:06:15, 4.96it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 1%|▏ | 269/20000 [01:19<1:06:05, 4.98it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 1%|▏ | 269/20000 [01:19<1:06:05, 4.98it/s, lr: 7.0e-05 loss: 9.320e-02] jam: 1%|▏ | 270/20000 [01:19<1:38:30, 3.34it/s, lr: 7.0e-05 loss: 9.320e-02] jam: 1%|▏ | 270/20000 [01:20<1:38:30, 3.34it/s, lr: 7.0e-05 loss: 1.451e-01] jam: 1%|▏ | 271/20000 [01:20<1:47:58, 3.05it/s, lr: 7.0e-05 loss: 1.451e-01] jam: 1%|▏ | 271/20000 [01:20<1:47:58, 3.05it/s, lr: 7.0e-05 loss: 1.639e-01] jam: 1%|▏ | 272/20000 [01:20<1:34:25, 3.48it/s, lr: 7.0e-05 loss: 1.639e-01] jam: 1%|▏ | 272/20000 [01:20<1:34:25, 3.48it/s, lr: 7.0e-05 loss: 4.360e-02] jam: 1%|▏ | 273/20000 [01:20<1:52:17, 2.93it/s, lr: 7.0e-05 loss: 4.360e-02] jam: 1%|▏ | 273/20000 [01:21<1:52:17, 2.93it/s, lr: 7.0e-05 loss: 7.987e-02] jam: 1%|▏ | 274/20000 [01:21<1:51:38, 2.94it/s, lr: 7.0e-05 loss: 7.987e-02] jam: 1%|▏ | 274/20000 [01:21<1:51:38, 2.94it/s, lr: 7.0e-05 loss: 1.340e-01] jam: 1%|▏ | 275/20000 [01:21<1:56:31, 2.82it/s, lr: 7.0e-05 loss: 1.340e-01] jam: 1%|▏ | 275/20000 [01:22<1:56:31, 2.82it/s, lr: 7.0e-05 loss: 4.187e-02] | |
| Saving at step 275 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000275.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 1%|▏ | 276/20000 [01:22<1:56:30, 2.82it/s, lr: 7.0e-05 loss: 9.165e-02] jam: 1%|▏ | 277/20000 [01:22<1:28:53, 3.70it/s, lr: 7.0e-05 loss: 9.165e-02] jam: 1%|▏ | 277/20000 [01:22<1:28:53, 3.70it/s, lr: 7.0e-05 loss: 2.246e-01] jam: 1%|▏ | 278/20000 [01:22<1:14:15, 4.43it/s, lr: 7.0e-05 loss: 2.246e-01] jam: 1%|▏ | 278/20000 [01:23<1:14:15, 4.43it/s, lr: 7.0e-05 loss: 1.423e-01] jam: 1%|▏ | 279/20000 [01:23<1:39:07, 3.32it/s, lr: 7.0e-05 loss: 1.423e-01] jam: 1%|▏ | 279/20000 [01:23<1:39:07, 3.32it/s, lr: 7.0e-05 loss: 1.082e-01] jam: 1%|▏ | 280/20000 [01:23<1:08:33, 4.79it/s, lr: 7.0e-05 loss: 1.082e-01] jam: 1%|▏ | 280/20000 [01:23<1:08:33, 4.79it/s, lr: 7.0e-05 loss: 1.809e-01] jam: 1%|▏ | 281/20000 [01:23<1:06:23, 4.95it/s, lr: 7.0e-05 loss: 1.809e-01] jam: 1%|▏ | 281/20000 [01:23<1:06:23, 4.95it/s, lr: 7.0e-05 loss: 7.994e-02] jam: 1%|▏ | 282/20000 [01:23<1:37:58, 3.35it/s, lr: 7.0e-05 loss: 7.994e-02] jam: 1%|▏ | 282/20000 [01:24<1:37:58, 3.35it/s, lr: 7.0e-05 loss: 8.888e-02] jam: 1%|▏ | 283/20000 [01:24<1:21:16, 4.04it/s, lr: 7.0e-05 loss: 8.888e-02] jam: 1%|▏ | 283/20000 [01:24<1:21:16, 4.04it/s, lr: 7.0e-05 loss: 1.179e-01] jam: 1%|▏ | 284/20000 [01:24<1:43:47, 3.17it/s, lr: 7.0e-05 loss: 1.179e-01] jam: 1%|▏ | 284/20000 [01:24<1:43:47, 3.17it/s, lr: 7.0e-05 loss: 6.604e-02] jam: 1%|▏ | 285/20000 [01:24<1:13:33, 4.47it/s, lr: 7.0e-05 loss: 6.604e-02] jam: 1%|▏ | 285/20000 [01:24<1:13:33, 4.47it/s, lr: 7.0e-05 loss: 1.856e-01] jam: 1%|▏ | 286/20000 [01:24<1:08:03, 4.83it/s, lr: 7.0e-05 loss: 1.856e-01] jam: 1%|▏ | 286/20000 [01:25<1:08:03, 4.83it/s, lr: 7.0e-05 loss: 6.442e-02] jam: 1%|▏ | 287/20000 [01:25<1:38:21, 3.34it/s, lr: 7.0e-05 loss: 6.442e-02] jam: 1%|▏ | 287/20000 [01:25<1:38:21, 3.34it/s, lr: 7.0e-05 loss: 1.040e-01] jam: 1%|▏ | 288/20000 [01:25<1:23:58, 3.91it/s, lr: 7.0e-05 loss: 1.040e-01] jam: 1%|▏ | 288/20000 [01:25<1:23:58, 3.91it/s, lr: 7.0e-05 loss: 1.264e-01] jam: 1%|▏ | 289/20000 [01:25<1:12:21, 4.54it/s, lr: 7.0e-05 loss: 1.264e-01] jam: 1%|▏ | 289/20000 [01:25<1:12:21, 4.54it/s, lr: 7.0e-05 loss: 1.172e-01] jam: 1%|▏ | 290/20000 [01:25<1:11:27, 4.60it/s, lr: 7.0e-05 loss: 1.172e-01] jam: 1%|▏ | 290/20000 [01:25<1:11:27, 4.60it/s, lr: 7.0e-05 loss: 2.040e-01] jam: 1%|▏ | 291/20000 [01:25<1:06:04, 4.97it/s, lr: 7.0e-05 loss: 2.040e-01] jam: 1%|▏ | 291/20000 [01:26<1:06:04, 4.97it/s, lr: 7.0e-05 loss: 1.419e-01] jam: 1%|▏ | 292/20000 [01:26<1:42:46, 3.20it/s, lr: 7.0e-05 loss: 1.419e-01] jam: 1%|▏ | 292/20000 [01:26<1:42:46, 3.20it/s, lr: 7.0e-05 loss: 4.276e-02] jam: 1%|▏ | 293/20000 [01:26<1:54:16, 2.87it/s, lr: 7.0e-05 loss: 4.276e-02] jam: 1%|▏ | 293/20000 [01:27<1:54:16, 2.87it/s, lr: 7.0e-05 loss: 1.713e-01] jam: 1%|▏ | 294/20000 [01:27<1:52:40, 2.92it/s, lr: 7.0e-05 loss: 1.713e-01] jam: 1%|▏ | 294/20000 [01:27<1:52:40, 2.92it/s, lr: 7.0e-05 loss: 8.705e-02] jam: 1%|▏ | 295/20000 [01:27<1:56:11, 2.83it/s, lr: 7.0e-05 loss: 8.705e-02] jam: 1%|▏ | 295/20000 [01:27<1:56:11, 2.83it/s, lr: 7.0e-05 loss: 1.632e-01] jam: 1%|▏ | 296/20000 [01:27<1:18:32, 4.18it/s, lr: 7.0e-05 loss: 1.632e-01] jam: 1%|▏ | 296/20000 [01:27<1:18:32, 4.18it/s, lr: 7.0e-05 loss: 1.704e-01] jam: 1%|▏ | 297/20000 [01:27<1:41:42, 3.23it/s, lr: 7.0e-05 loss: 1.704e-01] jam: 1%|▏ | 297/20000 [01:28<1:41:42, 3.23it/s, lr: 7.0e-05 loss: 1.490e-01] jam: 1%|▏ | 298/20000 [01:28<1:55:54, 2.83it/s, lr: 7.0e-05 loss: 1.490e-01] jam: 1%|▏ | 298/20000 [01:28<1:55:54, 2.83it/s, lr: 7.0e-05 loss: 1.601e-01] jam: 1%|▏ | 299/20000 [01:28<1:52:18, 2.92it/s, lr: 7.0e-05 loss: 1.601e-01] jam: 1%|▏ | 299/20000 [01:28<1:52:18, 2.92it/s, lr: 7.0e-05 loss: 2.001e-01] jam: 2%|▏ | 300/20000 [01:28<1:13:49, 4.45it/s, lr: 7.0e-05 loss: 2.001e-01] jam: 2%|▏ | 300/20000 [01:29<1:13:49, 4.45it/s, lr: 7.0e-05 loss: 2.040e-01] | |
| Saving at step 300 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000300.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 301/20000 [01:30<1:13:49, 4.45it/s, lr: 7.0e-05 loss: 1.537e-01] jam: 2%|▏ | 302/20000 [01:30<1:37:36, 3.36it/s, lr: 7.0e-05 loss: 1.537e-01] jam: 2%|▏ | 302/20000 [01:30<1:37:36, 3.36it/s, lr: 7.0e-05 loss: 1.748e-01] jam: 2%|▏ | 303/20000 [01:30<1:22:25, 3.98it/s, lr: 7.0e-05 loss: 1.748e-01] jam: 2%|▏ | 303/20000 [01:30<1:22:25, 3.98it/s, lr: 7.0e-05 loss: 6.036e-02] jam: 2%|▏ | 304/20000 [01:30<1:45:10, 3.12it/s, lr: 7.0e-05 loss: 6.036e-02] jam: 2%|▏ | 304/20000 [01:30<1:45:10, 3.12it/s, lr: 7.0e-05 loss: 1.517e-01] jam: 2%|▏ | 305/20000 [01:30<1:15:08, 4.37it/s, lr: 7.0e-05 loss: 1.517e-01] jam: 2%|▏ | 305/20000 [01:31<1:15:08, 4.37it/s, lr: 7.0e-05 loss: 2.076e-01] jam: 2%|▏ | 306/20000 [01:31<1:12:59, 4.50it/s, lr: 7.0e-05 loss: 2.076e-01] jam: 2%|▏ | 306/20000 [01:31<1:12:59, 4.50it/s, lr: 7.0e-05 loss: 1.811e-01] jam: 2%|▏ | 307/20000 [01:31<1:06:45, 4.92it/s, lr: 7.0e-05 loss: 1.811e-01] jam: 2%|▏ | 307/20000 [01:31<1:06:45, 4.92it/s, lr: 7.0e-05 loss: 1.439e-01] jam: 2%|▏ | 308/20000 [01:31<1:04:01, 5.13it/s, lr: 7.0e-05 loss: 1.439e-01] jam: 2%|▏ | 308/20000 [01:31<1:04:01, 5.13it/s, lr: 7.0e-05 loss: 1.186e-01] jam: 2%|▏ | 309/20000 [01:31<1:39:01, 3.31it/s, lr: 7.0e-05 loss: 1.186e-01] jam: 2%|▏ | 309/20000 [01:32<1:39:01, 3.31it/s, lr: 7.0e-05 loss: 2.115e-01] jam: 2%|▏ | 310/20000 [01:32<1:42:33, 3.20it/s, lr: 7.0e-05 loss: 2.115e-01] jam: 2%|▏ | 310/20000 [01:32<1:42:33, 3.20it/s, lr: 7.0e-05 loss: 1.871e-01] jam: 2%|▏ | 311/20000 [01:32<1:32:09, 3.56it/s, lr: 7.0e-05 loss: 1.871e-01] jam: 2%|▏ | 311/20000 [01:32<1:32:09, 3.56it/s, lr: 7.0e-05 loss: 7.017e-02] jam: 2%|▏ | 312/20000 [01:32<1:52:52, 2.91it/s, lr: 7.0e-05 loss: 7.017e-02] jam: 2%|▏ | 312/20000 [01:33<1:52:52, 2.91it/s, lr: 7.0e-05 loss: 1.498e-01] jam: 2%|▏ | 313/20000 [01:33<1:50:45, 2.96it/s, lr: 7.0e-05 loss: 1.498e-01] jam: 2%|▏ | 313/20000 [01:33<1:50:45, 2.96it/s, lr: 7.0e-05 loss: 8.232e-02] jam: 2%|▏ | 314/20000 [01:33<1:57:03, 2.80it/s, lr: 7.0e-05 loss: 8.232e-02] jam: 2%|▏ | 314/20000 [01:33<1:57:03, 2.80it/s, lr: 7.0e-05 loss: 1.272e-01] jam: 2%|▏ | 315/20000 [01:33<1:53:01, 2.90it/s, lr: 7.0e-05 loss: 1.272e-01] jam: 2%|▏ | 315/20000 [01:34<1:53:01, 2.90it/s, lr: 7.0e-05 loss: 6.491e-02] jam: 2%|▏ | 316/20000 [01:34<1:57:39, 2.79it/s, lr: 7.0e-05 loss: 6.491e-02] jam: 2%|▏ | 316/20000 [01:34<1:57:39, 2.79it/s, lr: 7.0e-05 loss: 2.631e-01] jam: 2%|▏ | 317/20000 [01:34<1:09:57, 4.69it/s, lr: 7.0e-05 loss: 2.631e-01] jam: 2%|▏ | 317/20000 [01:34<1:09:57, 4.69it/s, lr: 7.0e-05 loss: 7.361e-02] jam: 2%|▏ | 318/20000 [01:34<1:11:25, 4.59it/s, lr: 7.0e-05 loss: 7.361e-02] jam: 2%|▏ | 318/20000 [01:34<1:11:25, 4.59it/s, lr: 7.0e-05 loss: 8.325e-02] jam: 2%|▏ | 319/20000 [01:34<1:44:34, 3.14it/s, lr: 7.0e-05 loss: 8.325e-02] jam: 2%|▏ | 319/20000 [01:35<1:44:34, 3.14it/s, lr: 7.0e-05 loss: 1.528e-01] jam: 2%|▏ | 320/20000 [01:35<1:55:39, 2.84it/s, lr: 7.0e-05 loss: 1.528e-01] jam: 2%|▏ | 320/20000 [01:35<1:55:39, 2.84it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 2%|▏ | 321/20000 [01:35<1:17:03, 4.26it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 2%|▏ | 321/20000 [01:35<1:17:03, 4.26it/s, lr: 7.0e-05 loss: 1.775e-01] jam: 2%|▏ | 322/20000 [01:35<1:17:47, 4.22it/s, lr: 7.0e-05 loss: 1.775e-01] jam: 2%|▏ | 322/20000 [01:36<1:17:47, 4.22it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 2%|▏ | 323/20000 [01:36<1:49:39, 2.99it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 2%|▏ | 323/20000 [01:36<1:49:39, 2.99it/s, lr: 7.0e-05 loss: 4.051e-02] jam: 2%|▏ | 324/20000 [01:36<1:18:36, 4.17it/s, lr: 7.0e-05 loss: 4.051e-02] jam: 2%|▏ | 324/20000 [01:36<1:18:36, 4.17it/s, lr: 7.0e-05 loss: 1.331e-01] jam: 2%|▏ | 325/20000 [01:36<1:03:43, 5.15it/s, lr: 7.0e-05 loss: 1.331e-01] jam: 2%|▏ | 325/20000 [01:37<1:03:43, 5.15it/s, lr: 7.0e-05 loss: 8.336e-02] | |
| Saving at step 325 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000325.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 326/20000 [01:37<1:03:43, 5.15it/s, lr: 7.0e-05 loss: 7.592e-02] jam: 2%|▏ | 327/20000 [01:37<1:37:39, 3.36it/s, lr: 7.0e-05 loss: 7.592e-02] jam: 2%|▏ | 327/20000 [01:38<1:37:39, 3.36it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 2%|▏ | 328/20000 [01:38<1:52:36, 2.91it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 2%|▏ | 328/20000 [01:38<1:52:36, 2.91it/s, lr: 7.0e-05 loss: 1.183e-01] jam: 2%|▏ | 329/20000 [01:38<1:17:44, 4.22it/s, lr: 7.0e-05 loss: 1.183e-01] jam: 2%|▏ | 329/20000 [01:38<1:17:44, 4.22it/s, lr: 7.0e-05 loss: 1.290e-01] jam: 2%|▏ | 330/20000 [01:38<1:40:21, 3.27it/s, lr: 7.0e-05 loss: 1.290e-01] jam: 2%|▏ | 330/20000 [01:38<1:40:21, 3.27it/s, lr: 7.0e-05 loss: 1.014e-01] jam: 2%|▏ | 331/20000 [01:38<1:47:28, 3.05it/s, lr: 7.0e-05 loss: 1.014e-01] jam: 2%|▏ | 331/20000 [01:39<1:47:28, 3.05it/s, lr: 7.0e-05 loss: 3.644e-02] jam: 2%|▏ | 332/20000 [01:39<1:55:51, 2.83it/s, lr: 7.0e-05 loss: 3.644e-02] jam: 2%|▏ | 332/20000 [01:39<1:55:51, 2.83it/s, lr: 7.0e-05 loss: 1.013e-01] jam: 2%|▏ | 333/20000 [01:39<1:22:49, 3.96it/s, lr: 7.0e-05 loss: 1.013e-01] jam: 2%|▏ | 333/20000 [01:39<1:22:49, 3.96it/s, lr: 7.0e-05 loss: 1.216e-01] jam: 2%|▏ | 334/20000 [01:39<1:16:42, 4.27it/s, lr: 7.0e-05 loss: 1.216e-01] jam: 2%|▏ | 334/20000 [01:39<1:16:42, 4.27it/s, lr: 7.0e-05 loss: 2.338e-01] jam: 2%|▏ | 335/20000 [01:39<1:11:59, 4.55it/s, lr: 7.0e-05 loss: 2.338e-01] jam: 2%|▏ | 335/20000 [01:40<1:11:59, 4.55it/s, lr: 7.0e-05 loss: 2.512e-01] jam: 2%|▏ | 336/20000 [01:40<1:10:24, 4.65it/s, lr: 7.0e-05 loss: 2.512e-01] jam: 2%|▏ | 336/20000 [01:40<1:10:24, 4.65it/s, lr: 7.0e-05 loss: 9.107e-02] jam: 2%|▏ | 337/20000 [01:40<1:06:53, 4.90it/s, lr: 7.0e-05 loss: 9.107e-02] jam: 2%|▏ | 337/20000 [01:40<1:06:53, 4.90it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 2%|▏ | 338/20000 [01:40<1:38:21, 3.33it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 2%|▏ | 338/20000 [01:41<1:38:21, 3.33it/s, lr: 7.0e-05 loss: 1.687e-01] jam: 2%|▏ | 339/20000 [01:41<1:55:45, 2.83it/s, lr: 7.0e-05 loss: 1.687e-01] jam: 2%|▏ | 339/20000 [01:41<1:55:45, 2.83it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 2%|▏ | 340/20000 [01:41<1:52:48, 2.90it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 2%|▏ | 340/20000 [01:41<1:52:48, 2.90it/s, lr: 7.0e-05 loss: 1.854e-01] jam: 2%|▏ | 341/20000 [01:41<1:55:59, 2.82it/s, lr: 7.0e-05 loss: 1.854e-01] jam: 2%|▏ | 341/20000 [01:42<1:55:59, 2.82it/s, lr: 7.0e-05 loss: 7.304e-02] jam: 2%|▏ | 342/20000 [01:42<1:51:45, 2.93it/s, lr: 7.0e-05 loss: 7.304e-02] jam: 2%|▏ | 342/20000 [01:42<1:51:45, 2.93it/s, lr: 7.0e-05 loss: 2.365e-01] jam: 2%|▏ | 343/20000 [01:42<1:35:29, 3.43it/s, lr: 7.0e-05 loss: 2.365e-01] jam: 2%|▏ | 343/20000 [01:42<1:35:29, 3.43it/s, lr: 7.0e-05 loss: 8.303e-02] jam: 2%|▏ | 344/20000 [01:42<1:45:52, 3.09it/s, lr: 7.0e-05 loss: 8.303e-02] jam: 2%|▏ | 344/20000 [01:42<1:45:52, 3.09it/s, lr: 7.0e-05 loss: 1.851e-01] jam: 2%|▏ | 345/20000 [01:42<1:22:18, 3.98it/s, lr: 7.0e-05 loss: 1.851e-01] jam: 2%|▏ | 345/20000 [01:43<1:22:18, 3.98it/s, lr: 7.0e-05 loss: 3.311e-01] jam: 2%|▏ | 346/20000 [01:43<1:15:41, 4.33it/s, lr: 7.0e-05 loss: 3.311e-01] jam: 2%|▏ | 346/20000 [01:43<1:15:41, 4.33it/s, lr: 7.0e-05 loss: 1.006e-01] jam: 2%|▏ | 347/20000 [01:43<1:07:10, 4.88it/s, lr: 7.0e-05 loss: 1.006e-01] jam: 2%|▏ | 347/20000 [01:43<1:07:10, 4.88it/s, lr: 7.0e-05 loss: 1.662e-01] jam: 2%|▏ | 348/20000 [01:43<1:05:30, 5.00it/s, lr: 7.0e-05 loss: 1.662e-01] jam: 2%|▏ | 348/20000 [01:43<1:05:30, 5.00it/s, lr: 7.0e-05 loss: 2.366e-01] jam: 2%|▏ | 349/20000 [01:43<1:12:36, 4.51it/s, lr: 7.0e-05 loss: 2.366e-01] jam: 2%|▏ | 349/20000 [01:43<1:12:36, 4.51it/s, lr: 7.0e-05 loss: 2.886e-01] jam: 2%|▏ | 350/20000 [01:43<1:11:53, 4.56it/s, lr: 7.0e-05 loss: 2.886e-01] jam: 2%|▏ | 350/20000 [01:44<1:11:53, 4.56it/s, lr: 7.0e-05 loss: 2.584e-01] | |
| Saving at step 350 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000350.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 351/20000 [01:45<1:11:53, 4.56it/s, lr: 7.0e-05 loss: 1.310e-01] jam: 2%|▏ | 352/20000 [01:45<1:33:48, 3.49it/s, lr: 7.0e-05 loss: 1.310e-01] jam: 2%|▏ | 352/20000 [01:45<1:33:48, 3.49it/s, lr: 7.0e-05 loss: 1.637e-01] jam: 2%|▏ | 353/20000 [01:45<1:28:18, 3.71it/s, lr: 7.0e-05 loss: 1.637e-01] jam: 2%|▏ | 353/20000 [01:45<1:28:18, 3.71it/s, lr: 7.0e-05 loss: 2.359e-01] jam: 2%|▏ | 354/20000 [01:45<1:43:15, 3.17it/s, lr: 7.0e-05 loss: 2.359e-01] jam: 2%|▏ | 354/20000 [01:46<1:43:15, 3.17it/s, lr: 7.0e-05 loss: 2.247e-01] jam: 2%|▏ | 355/20000 [01:46<1:53:23, 2.89it/s, lr: 7.0e-05 loss: 2.247e-01] jam: 2%|▏ | 355/20000 [01:46<1:53:23, 2.89it/s, lr: 7.0e-05 loss: 1.640e-01] jam: 2%|▏ | 356/20000 [01:46<1:52:08, 2.92it/s, lr: 7.0e-05 loss: 1.640e-01] jam: 2%|▏ | 356/20000 [01:46<1:52:08, 2.92it/s, lr: 7.0e-05 loss: 7.839e-02] jam: 2%|▏ | 357/20000 [01:46<1:22:02, 3.99it/s, lr: 7.0e-05 loss: 7.839e-02] jam: 2%|▏ | 357/20000 [01:46<1:22:02, 3.99it/s, lr: 7.0e-05 loss: 1.923e-01] jam: 2%|▏ | 358/20000 [01:46<1:13:52, 4.43it/s, lr: 7.0e-05 loss: 1.923e-01] jam: 2%|▏ | 358/20000 [01:47<1:13:52, 4.43it/s, lr: 7.0e-05 loss: 7.458e-02] jam: 2%|▏ | 359/20000 [01:47<1:07:50, 4.83it/s, lr: 7.0e-05 loss: 7.458e-02] jam: 2%|▏ | 359/20000 [01:47<1:07:50, 4.83it/s, lr: 7.0e-05 loss: 1.260e-01] jam: 2%|▏ | 360/20000 [01:47<1:40:29, 3.26it/s, lr: 7.0e-05 loss: 1.260e-01] jam: 2%|▏ | 360/20000 [01:47<1:40:29, 3.26it/s, lr: 7.0e-05 loss: 2.771e-01] jam: 2%|▏ | 361/20000 [01:47<1:22:21, 3.97it/s, lr: 7.0e-05 loss: 2.771e-01] jam: 2%|▏ | 361/20000 [01:47<1:22:21, 3.97it/s, lr: 7.0e-05 loss: 7.712e-02] jam: 2%|▏ | 362/20000 [01:47<1:13:38, 4.44it/s, lr: 7.0e-05 loss: 7.712e-02] jam: 2%|▏ | 362/20000 [01:48<1:13:38, 4.44it/s, lr: 7.0e-05 loss: 2.494e-01] jam: 2%|▏ | 363/20000 [01:48<1:07:35, 4.84it/s, lr: 7.0e-05 loss: 2.494e-01] jam: 2%|▏ | 363/20000 [01:48<1:07:35, 4.84it/s, lr: 7.0e-05 loss: 1.969e-01] jam: 2%|▏ | 364/20000 [01:48<1:42:16, 3.20it/s, lr: 7.0e-05 loss: 1.969e-01] jam: 2%|▏ | 364/20000 [01:48<1:42:16, 3.20it/s, lr: 7.0e-05 loss: 1.189e-01] jam: 2%|▏ | 365/20000 [01:48<1:22:28, 3.97it/s, lr: 7.0e-05 loss: 1.189e-01] jam: 2%|▏ | 365/20000 [01:48<1:22:28, 3.97it/s, lr: 7.0e-05 loss: 1.277e-01] jam: 2%|▏ | 366/20000 [01:48<1:16:38, 4.27it/s, lr: 7.0e-05 loss: 1.277e-01] jam: 2%|▏ | 366/20000 [01:49<1:16:38, 4.27it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 2%|▏ | 367/20000 [01:49<1:42:36, 3.19it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 2%|▏ | 367/20000 [01:49<1:42:36, 3.19it/s, lr: 7.0e-05 loss: 1.746e-01] jam: 2%|▏ | 368/20000 [01:49<1:23:13, 3.93it/s, lr: 7.0e-05 loss: 1.746e-01] jam: 2%|▏ | 368/20000 [01:49<1:23:13, 3.93it/s, lr: 7.0e-05 loss: 7.584e-02] jam: 2%|▏ | 369/20000 [01:49<1:10:33, 4.64it/s, lr: 7.0e-05 loss: 7.584e-02] jam: 2%|▏ | 369/20000 [01:49<1:10:33, 4.64it/s, lr: 7.0e-05 loss: 7.131e-02] jam: 2%|▏ | 370/20000 [01:49<1:45:03, 3.11it/s, lr: 7.0e-05 loss: 7.131e-02] jam: 2%|▏ | 370/20000 [01:50<1:45:03, 3.11it/s, lr: 7.0e-05 loss: 2.158e-01] jam: 2%|▏ | 371/20000 [01:50<1:49:36, 2.98it/s, lr: 7.0e-05 loss: 2.158e-01] jam: 2%|▏ | 371/20000 [01:50<1:49:36, 2.98it/s, lr: 7.0e-05 loss: 1.177e-01] jam: 2%|▏ | 372/20000 [01:50<1:55:55, 2.82it/s, lr: 7.0e-05 loss: 1.177e-01] jam: 2%|▏ | 372/20000 [01:50<1:55:55, 2.82it/s, lr: 7.0e-05 loss: 8.523e-02] jam: 2%|▏ | 373/20000 [01:50<1:11:36, 4.57it/s, lr: 7.0e-05 loss: 8.523e-02] jam: 2%|▏ | 373/20000 [01:51<1:11:36, 4.57it/s, lr: 7.0e-05 loss: 1.013e-01] jam: 2%|▏ | 374/20000 [01:51<1:05:30, 4.99it/s, lr: 7.0e-05 loss: 1.013e-01] jam: 2%|▏ | 374/20000 [01:51<1:05:30, 4.99it/s, lr: 7.0e-05 loss: 1.101e-01] jam: 2%|▏ | 375/20000 [01:51<1:04:27, 5.07it/s, lr: 7.0e-05 loss: 1.101e-01] jam: 2%|▏ | 375/20000 [01:51<1:04:27, 5.07it/s, lr: 7.0e-05 loss: 1.500e-01] | |
| Saving at step 375 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000375.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 376/20000 [01:52<1:04:27, 5.07it/s, lr: 7.0e-05 loss: 1.395e-01] jam: 2%|▏ | 377/20000 [01:52<1:09:05, 4.73it/s, lr: 7.0e-05 loss: 1.395e-01] jam: 2%|▏ | 377/20000 [01:52<1:09:05, 4.73it/s, lr: 7.0e-05 loss: 2.454e-02] jam: 2%|▏ | 378/20000 [01:52<1:33:36, 3.49it/s, lr: 7.0e-05 loss: 2.454e-02] jam: 2%|▏ | 378/20000 [01:52<1:33:36, 3.49it/s, lr: 7.0e-05 loss: 1.144e-01] jam: 2%|▏ | 379/20000 [01:52<1:49:08, 3.00it/s, lr: 7.0e-05 loss: 1.144e-01] jam: 2%|▏ | 379/20000 [01:53<1:49:08, 3.00it/s, lr: 7.0e-05 loss: 8.786e-02] jam: 2%|▏ | 380/20000 [01:53<1:50:34, 2.96it/s, lr: 7.0e-05 loss: 8.786e-02] jam: 2%|▏ | 380/20000 [01:53<1:50:34, 2.96it/s, lr: 7.0e-05 loss: 5.962e-02] jam: 2%|▏ | 381/20000 [01:53<1:51:07, 2.94it/s, lr: 7.0e-05 loss: 5.962e-02] jam: 2%|▏ | 381/20000 [01:53<1:51:07, 2.94it/s, lr: 7.0e-05 loss: 8.307e-02] jam: 2%|▏ | 382/20000 [01:53<1:51:03, 2.94it/s, lr: 7.0e-05 loss: 8.307e-02] jam: 2%|▏ | 382/20000 [01:54<1:51:03, 2.94it/s, lr: 7.0e-05 loss: 6.489e-02] jam: 2%|▏ | 383/20000 [01:54<1:55:32, 2.83it/s, lr: 7.0e-05 loss: 6.489e-02] jam: 2%|▏ | 383/20000 [01:54<1:55:32, 2.83it/s, lr: 7.0e-05 loss: 1.784e-01] jam: 2%|▏ | 384/20000 [01:54<1:18:46, 4.15it/s, lr: 7.0e-05 loss: 1.784e-01] jam: 2%|▏ | 384/20000 [01:54<1:18:46, 4.15it/s, lr: 7.0e-05 loss: 1.038e-01] jam: 2%|▏ | 385/20000 [01:54<1:40:56, 3.24it/s, lr: 7.0e-05 loss: 1.038e-01] jam: 2%|▏ | 385/20000 [01:54<1:40:56, 3.24it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 2%|▏ | 386/20000 [01:54<1:19:44, 4.10it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 2%|▏ | 386/20000 [01:55<1:19:44, 4.10it/s, lr: 7.0e-05 loss: 1.199e-01] jam: 2%|▏ | 387/20000 [01:55<1:45:03, 3.11it/s, lr: 7.0e-05 loss: 1.199e-01] jam: 2%|▏ | 387/20000 [01:55<1:45:03, 3.11it/s, lr: 7.0e-05 loss: 1.247e-01] jam: 2%|▏ | 388/20000 [01:55<1:49:17, 2.99it/s, lr: 7.0e-05 loss: 1.247e-01] jam: 2%|▏ | 388/20000 [01:56<1:49:17, 2.99it/s, lr: 7.0e-05 loss: 7.214e-02] jam: 2%|▏ | 389/20000 [01:56<1:56:15, 2.81it/s, lr: 7.0e-05 loss: 7.214e-02] jam: 2%|▏ | 389/20000 [01:56<1:56:15, 2.81it/s, lr: 7.0e-05 loss: 1.261e-01] jam: 2%|▏ | 390/20000 [01:56<1:52:33, 2.90it/s, lr: 7.0e-05 loss: 1.261e-01] jam: 2%|▏ | 390/20000 [01:56<1:52:33, 2.90it/s, lr: 7.0e-05 loss: 1.021e-01] jam: 2%|▏ | 391/20000 [01:56<1:56:11, 2.81it/s, lr: 7.0e-05 loss: 1.021e-01] jam: 2%|▏ | 391/20000 [01:56<1:56:11, 2.81it/s, lr: 7.0e-05 loss: 2.431e-01] jam: 2%|▏ | 392/20000 [01:56<1:17:22, 4.22it/s, lr: 7.0e-05 loss: 2.431e-01] jam: 2%|▏ | 392/20000 [01:57<1:17:22, 4.22it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 2%|▏ | 393/20000 [01:57<1:08:32, 4.77it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 2%|▏ | 393/20000 [01:57<1:08:32, 4.77it/s, lr: 7.0e-05 loss: 1.663e-01] jam: 2%|▏ | 394/20000 [01:57<1:05:46, 4.97it/s, lr: 7.0e-05 loss: 1.663e-01] jam: 2%|▏ | 394/20000 [01:57<1:05:46, 4.97it/s, lr: 7.0e-05 loss: 1.099e-01] jam: 2%|▏ | 395/20000 [01:57<1:38:10, 3.33it/s, lr: 7.0e-05 loss: 1.099e-01] jam: 2%|▏ | 395/20000 [01:57<1:38:10, 3.33it/s, lr: 7.0e-05 loss: 2.052e-01] jam: 2%|▏ | 396/20000 [01:57<1:06:59, 4.88it/s, lr: 7.0e-05 loss: 2.052e-01] jam: 2%|▏ | 396/20000 [01:57<1:06:59, 4.88it/s, lr: 7.0e-05 loss: 2.562e-01] jam: 2%|▏ | 397/20000 [01:57<1:00:28, 5.40it/s, lr: 7.0e-05 loss: 2.562e-01] jam: 2%|▏ | 397/20000 [01:58<1:00:28, 5.40it/s, lr: 7.0e-05 loss: 1.475e-01] jam: 2%|▏ | 398/20000 [01:58<1:35:44, 3.41it/s, lr: 7.0e-05 loss: 1.475e-01] jam: 2%|▏ | 398/20000 [01:58<1:35:44, 3.41it/s, lr: 7.0e-05 loss: 4.632e-02] jam: 2%|▏ | 399/20000 [01:58<1:46:23, 3.07it/s, lr: 7.0e-05 loss: 4.632e-02] jam: 2%|▏ | 399/20000 [01:59<1:46:23, 3.07it/s, lr: 7.0e-05 loss: 2.468e-01] jam: 2%|▏ | 400/20000 [01:59<1:52:40, 2.90it/s, lr: 7.0e-05 loss: 2.468e-01] jam: 2%|▏ | 400/20000 [01:59<1:52:40, 2.90it/s, lr: 7.0e-05 loss: 2.111e-01] | |
| Saving at step 400 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000400.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 401/20000 [02:00<1:52:40, 2.90it/s, lr: 7.0e-05 loss: 6.475e-02] jam: 2%|▏ | 402/20000 [02:00<1:19:57, 4.08it/s, lr: 7.0e-05 loss: 6.475e-02] jam: 2%|▏ | 402/20000 [02:00<1:19:57, 4.08it/s, lr: 7.0e-05 loss: 7.380e-02] jam: 2%|▏ | 403/20000 [02:00<1:08:57, 4.74it/s, lr: 7.0e-05 loss: 7.380e-02] jam: 2%|▏ | 403/20000 [02:00<1:08:57, 4.74it/s, lr: 7.0e-05 loss: 2.693e-01] jam: 2%|▏ | 404/20000 [02:00<1:11:12, 4.59it/s, lr: 7.0e-05 loss: 2.693e-01] jam: 2%|▏ | 404/20000 [02:00<1:11:12, 4.59it/s, lr: 7.0e-05 loss: 1.851e-01] jam: 2%|▏ | 405/20000 [02:00<1:03:06, 5.17it/s, lr: 7.0e-05 loss: 1.851e-01] jam: 2%|▏ | 405/20000 [02:00<1:03:06, 5.17it/s, lr: 7.0e-05 loss: 1.180e-01] jam: 2%|▏ | 406/20000 [02:00<1:18:46, 4.15it/s, lr: 7.0e-05 loss: 1.180e-01] jam: 2%|▏ | 406/20000 [02:01<1:18:46, 4.15it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 2%|▏ | 407/20000 [02:01<1:01:17, 5.33it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 2%|▏ | 407/20000 [02:01<1:01:17, 5.33it/s, lr: 7.0e-05 loss: 1.683e-01] jam: 2%|▏ | 408/20000 [02:01<1:36:21, 3.39it/s, lr: 7.0e-05 loss: 1.683e-01] jam: 2%|▏ | 408/20000 [02:01<1:36:21, 3.39it/s, lr: 7.0e-05 loss: 9.119e-02] jam: 2%|▏ | 409/20000 [02:01<1:12:07, 4.53it/s, lr: 7.0e-05 loss: 9.119e-02] jam: 2%|▏ | 409/20000 [02:01<1:12:07, 4.53it/s, lr: 7.0e-05 loss: 1.527e-01] jam: 2%|▏ | 410/20000 [02:01<1:44:00, 3.14it/s, lr: 7.0e-05 loss: 1.527e-01] jam: 2%|▏ | 410/20000 [02:02<1:44:00, 3.14it/s, lr: 7.0e-05 loss: 2.723e-01] jam: 2%|▏ | 411/20000 [02:02<1:13:52, 4.42it/s, lr: 7.0e-05 loss: 2.723e-01] jam: 2%|▏ | 411/20000 [02:02<1:13:52, 4.42it/s, lr: 7.0e-05 loss: 1.871e-01] jam: 2%|▏ | 412/20000 [02:02<1:42:13, 3.19it/s, lr: 7.0e-05 loss: 1.871e-01] jam: 2%|▏ | 412/20000 [02:02<1:42:13, 3.19it/s, lr: 7.0e-05 loss: 1.547e-01] jam: 2%|▏ | 413/20000 [02:02<1:22:58, 3.93it/s, lr: 7.0e-05 loss: 1.547e-01] jam: 2%|▏ | 413/20000 [02:02<1:22:58, 3.93it/s, lr: 7.0e-05 loss: 1.592e-01] jam: 2%|▏ | 414/20000 [02:02<1:09:09, 4.72it/s, lr: 7.0e-05 loss: 1.592e-01] jam: 2%|▏ | 414/20000 [02:03<1:09:09, 4.72it/s, lr: 7.0e-05 loss: 1.796e-01] jam: 2%|▏ | 415/20000 [02:03<1:43:27, 3.15it/s, lr: 7.0e-05 loss: 1.796e-01] jam: 2%|▏ | 415/20000 [02:03<1:43:27, 3.15it/s, lr: 7.0e-05 loss: 8.435e-02] jam: 2%|▏ | 416/20000 [02:03<1:48:45, 3.00it/s, lr: 7.0e-05 loss: 8.435e-02] jam: 2%|▏ | 416/20000 [02:03<1:48:45, 3.00it/s, lr: 7.0e-05 loss: 4.942e-02] jam: 2%|▏ | 417/20000 [02:03<1:54:29, 2.85it/s, lr: 7.0e-05 loss: 4.942e-02] jam: 2%|▏ | 417/20000 [02:04<1:54:29, 2.85it/s, lr: 7.0e-05 loss: 1.697e-01] jam: 2%|▏ | 418/20000 [02:04<1:52:17, 2.91it/s, lr: 7.0e-05 loss: 1.697e-01] jam: 2%|▏ | 418/20000 [02:04<1:52:17, 2.91it/s, lr: 7.0e-05 loss: 1.895e-01] jam: 2%|▏ | 419/20000 [02:04<1:25:03, 3.84it/s, lr: 7.0e-05 loss: 1.895e-01] jam: 2%|▏ | 419/20000 [02:04<1:25:03, 3.84it/s, lr: 7.0e-05 loss: 2.299e-01] jam: 2%|▏ | 420/20000 [02:04<1:12:01, 4.53it/s, lr: 7.0e-05 loss: 2.299e-01] jam: 2%|▏ | 420/20000 [02:04<1:12:01, 4.53it/s, lr: 7.0e-05 loss: 1.358e-01] jam: 2%|▏ | 421/20000 [02:04<1:06:03, 4.94it/s, lr: 7.0e-05 loss: 1.358e-01] jam: 2%|▏ | 421/20000 [02:05<1:06:03, 4.94it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 2%|▏ | 422/20000 [02:05<1:11:49, 4.54it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 2%|▏ | 422/20000 [02:05<1:11:49, 4.54it/s, lr: 7.0e-05 loss: 1.190e-01] jam: 2%|▏ | 423/20000 [02:05<1:01:05, 5.34it/s, lr: 7.0e-05 loss: 1.190e-01] jam: 2%|▏ | 423/20000 [02:05<1:01:05, 5.34it/s, lr: 7.0e-05 loss: 8.495e-02] jam: 2%|▏ | 424/20000 [02:05<1:00:31, 5.39it/s, lr: 7.0e-05 loss: 8.495e-02] jam: 2%|▏ | 424/20000 [02:05<1:00:31, 5.39it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 2%|▏ | 425/20000 [02:05<1:37:28, 3.35it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 2%|▏ | 425/20000 [02:06<1:37:28, 3.35it/s, lr: 7.0e-05 loss: 1.146e-01] | |
| Saving at step 425 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000425.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 426/20000 [02:06<1:37:28, 3.35it/s, lr: 7.0e-05 loss: 2.059e-01] jam: 2%|▏ | 427/20000 [02:06<1:36:52, 3.37it/s, lr: 7.0e-05 loss: 2.059e-01] jam: 2%|▏ | 427/20000 [02:07<1:36:52, 3.37it/s, lr: 7.0e-05 loss: 9.923e-02] jam: 2%|▏ | 428/20000 [02:07<1:17:56, 4.19it/s, lr: 7.0e-05 loss: 9.923e-02] jam: 2%|▏ | 428/20000 [02:07<1:17:56, 4.19it/s, lr: 7.0e-05 loss: 1.647e-01] jam: 2%|▏ | 429/20000 [02:07<1:06:41, 4.89it/s, lr: 7.0e-05 loss: 1.647e-01] jam: 2%|▏ | 429/20000 [02:07<1:06:41, 4.89it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 2%|▏ | 430/20000 [02:07<1:05:30, 4.98it/s, lr: 7.0e-05 loss: 1.627e-01] jam: 2%|▏ | 430/20000 [02:07<1:05:30, 4.98it/s, lr: 7.0e-05 loss: 1.359e-01] jam: 2%|▏ | 431/20000 [02:07<1:45:13, 3.10it/s, lr: 7.0e-05 loss: 1.359e-01] jam: 2%|▏ | 431/20000 [02:08<1:45:13, 3.10it/s, lr: 7.0e-05 loss: 1.211e-01] jam: 2%|▏ | 432/20000 [02:08<1:13:37, 4.43it/s, lr: 7.0e-05 loss: 1.211e-01] jam: 2%|▏ | 432/20000 [02:08<1:13:37, 4.43it/s, lr: 7.0e-05 loss: 1.087e-01] jam: 2%|▏ | 433/20000 [02:08<1:07:18, 4.84it/s, lr: 7.0e-05 loss: 1.087e-01] jam: 2%|▏ | 433/20000 [02:08<1:07:18, 4.84it/s, lr: 7.0e-05 loss: 2.927e-01] jam: 2%|▏ | 434/20000 [02:08<1:38:13, 3.32it/s, lr: 7.0e-05 loss: 2.927e-01] jam: 2%|▏ | 434/20000 [02:08<1:38:13, 3.32it/s, lr: 7.0e-05 loss: 1.542e-01] jam: 2%|▏ | 435/20000 [02:08<1:54:46, 2.84it/s, lr: 7.0e-05 loss: 1.542e-01] jam: 2%|▏ | 435/20000 [02:09<1:54:46, 2.84it/s, lr: 7.0e-05 loss: 2.317e-02] jam: 2%|▏ | 436/20000 [02:09<1:18:03, 4.18it/s, lr: 7.0e-05 loss: 2.317e-02] jam: 2%|▏ | 436/20000 [02:09<1:18:03, 4.18it/s, lr: 7.0e-05 loss: 1.677e-01] jam: 2%|▏ | 437/20000 [02:09<1:06:11, 4.93it/s, lr: 7.0e-05 loss: 1.677e-01] jam: 2%|▏ | 437/20000 [02:09<1:06:11, 4.93it/s, lr: 7.0e-05 loss: 7.244e-02] jam: 2%|▏ | 438/20000 [02:09<1:06:21, 4.91it/s, lr: 7.0e-05 loss: 7.244e-02] jam: 2%|▏ | 438/20000 [02:09<1:06:21, 4.91it/s, lr: 7.0e-05 loss: 6.652e-02] jam: 2%|▏ | 439/20000 [02:09<1:37:48, 3.33it/s, lr: 7.0e-05 loss: 6.652e-02] jam: 2%|▏ | 439/20000 [02:10<1:37:48, 3.33it/s, lr: 7.0e-05 loss: 7.901e-02] jam: 2%|▏ | 440/20000 [02:10<1:55:20, 2.83it/s, lr: 7.0e-05 loss: 7.901e-02] jam: 2%|▏ | 440/20000 [02:10<1:55:20, 2.83it/s, lr: 7.0e-05 loss: 1.613e-01] jam: 2%|▏ | 441/20000 [02:10<1:25:22, 3.82it/s, lr: 7.0e-05 loss: 1.613e-01] jam: 2%|▏ | 441/20000 [02:10<1:25:22, 3.82it/s, lr: 7.0e-05 loss: 1.457e-01] jam: 2%|▏ | 442/20000 [02:10<1:13:49, 4.42it/s, lr: 7.0e-05 loss: 1.457e-01] jam: 2%|▏ | 442/20000 [02:11<1:13:49, 4.42it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 2%|▏ | 443/20000 [02:11<1:45:34, 3.09it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 2%|▏ | 443/20000 [02:11<1:45:34, 3.09it/s, lr: 7.0e-05 loss: 5.803e-02] jam: 2%|▏ | 444/20000 [02:11<1:24:39, 3.85it/s, lr: 7.0e-05 loss: 5.803e-02] jam: 2%|▏ | 444/20000 [02:11<1:24:39, 3.85it/s, lr: 7.0e-05 loss: 2.369e-01] jam: 2%|▏ | 445/20000 [02:11<1:11:18, 4.57it/s, lr: 7.0e-05 loss: 2.369e-01] jam: 2%|▏ | 445/20000 [02:11<1:11:18, 4.57it/s, lr: 7.0e-05 loss: 2.165e-01] jam: 2%|▏ | 446/20000 [02:11<1:06:30, 4.90it/s, lr: 7.0e-05 loss: 2.165e-01] jam: 2%|▏ | 446/20000 [02:12<1:06:30, 4.90it/s, lr: 7.0e-05 loss: 1.093e-01] jam: 2%|▏ | 447/20000 [02:12<1:43:55, 3.14it/s, lr: 7.0e-05 loss: 1.093e-01] jam: 2%|▏ | 447/20000 [02:12<1:43:55, 3.14it/s, lr: 7.0e-05 loss: 1.472e-01] jam: 2%|▏ | 448/20000 [02:12<1:48:30, 3.00it/s, lr: 7.0e-05 loss: 1.472e-01] jam: 2%|▏ | 448/20000 [02:12<1:48:30, 3.00it/s, lr: 7.0e-05 loss: 8.551e-02] jam: 2%|▏ | 449/20000 [02:12<1:18:26, 4.15it/s, lr: 7.0e-05 loss: 8.551e-02] jam: 2%|▏ | 449/20000 [02:12<1:18:26, 4.15it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 2%|▏ | 450/20000 [02:12<1:09:06, 4.71it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 2%|▏ | 450/20000 [02:13<1:09:06, 4.71it/s, lr: 7.0e-05 loss: 6.021e-02] | |
| Saving at step 450 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000450.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 451/20000 [02:13<1:09:06, 4.71it/s, lr: 7.0e-05 loss: 8.581e-02] jam: 2%|▏ | 452/20000 [02:13<1:17:45, 4.19it/s, lr: 7.0e-05 loss: 8.581e-02] jam: 2%|▏ | 452/20000 [02:14<1:17:45, 4.19it/s, lr: 7.0e-05 loss: 6.685e-02] jam: 2%|▏ | 453/20000 [02:14<1:37:43, 3.33it/s, lr: 7.0e-05 loss: 6.685e-02] jam: 2%|▏ | 453/20000 [02:14<1:37:43, 3.33it/s, lr: 7.0e-05 loss: 1.854e-01] jam: 2%|▏ | 454/20000 [02:14<1:54:17, 2.85it/s, lr: 7.0e-05 loss: 1.854e-01] jam: 2%|▏ | 454/20000 [02:15<1:54:17, 2.85it/s, lr: 7.0e-05 loss: 1.265e-01] jam: 2%|▏ | 455/20000 [02:15<1:51:54, 2.91it/s, lr: 7.0e-05 loss: 1.265e-01] jam: 2%|▏ | 455/20000 [02:15<1:51:54, 2.91it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 2%|▏ | 456/20000 [02:15<1:54:43, 2.84it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 2%|▏ | 456/20000 [02:15<1:54:43, 2.84it/s, lr: 7.0e-05 loss: 1.012e-01] jam: 2%|▏ | 457/20000 [02:15<1:51:47, 2.91it/s, lr: 7.0e-05 loss: 1.012e-01] jam: 2%|▏ | 457/20000 [02:16<1:51:47, 2.91it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 2%|▏ | 458/20000 [02:16<1:56:23, 2.80it/s, lr: 7.0e-05 loss: 1.793e-01] jam: 2%|▏ | 458/20000 [02:16<1:56:23, 2.80it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 2%|▏ | 459/20000 [02:16<1:52:10, 2.90it/s, lr: 7.0e-05 loss: 1.971e-01] jam: 2%|▏ | 459/20000 [02:16<1:52:10, 2.90it/s, lr: 7.0e-05 loss: 1.967e-01] jam: 2%|▏ | 460/20000 [02:16<1:17:52, 4.18it/s, lr: 7.0e-05 loss: 1.967e-01] jam: 2%|▏ | 460/20000 [02:16<1:17:52, 4.18it/s, lr: 7.0e-05 loss: 2.521e-01] jam: 2%|▏ | 461/20000 [02:16<1:10:23, 4.63it/s, lr: 7.0e-05 loss: 2.521e-01] jam: 2%|▏ | 461/20000 [02:17<1:10:23, 4.63it/s, lr: 7.0e-05 loss: 3.017e-01] jam: 2%|▏ | 462/20000 [02:17<1:17:05, 4.22it/s, lr: 7.0e-05 loss: 3.017e-01] jam: 2%|▏ | 462/20000 [02:17<1:17:05, 4.22it/s, lr: 7.0e-05 loss: 3.189e-01] jam: 2%|▏ | 463/20000 [02:17<1:43:14, 3.15it/s, lr: 7.0e-05 loss: 3.189e-01] jam: 2%|▏ | 463/20000 [02:17<1:43:14, 3.15it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 2%|▏ | 464/20000 [02:17<1:20:08, 4.06it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 2%|▏ | 464/20000 [02:17<1:20:08, 4.06it/s, lr: 7.0e-05 loss: 9.529e-02] jam: 2%|▏ | 465/20000 [02:17<1:18:58, 4.12it/s, lr: 7.0e-05 loss: 9.529e-02] jam: 2%|▏ | 465/20000 [02:18<1:18:58, 4.12it/s, lr: 7.0e-05 loss: 2.442e-01] jam: 2%|▏ | 466/20000 [02:18<1:09:12, 4.70it/s, lr: 7.0e-05 loss: 2.442e-01] jam: 2%|▏ | 466/20000 [02:18<1:09:12, 4.70it/s, lr: 7.0e-05 loss: 6.206e-02] jam: 2%|▏ | 467/20000 [02:18<1:38:44, 3.30it/s, lr: 7.0e-05 loss: 6.206e-02] jam: 2%|▏ | 467/20000 [02:18<1:38:44, 3.30it/s, lr: 7.0e-05 loss: 7.180e-02] jam: 2%|▏ | 468/20000 [02:18<1:13:03, 4.46it/s, lr: 7.0e-05 loss: 7.180e-02] jam: 2%|▏ | 468/20000 [02:18<1:13:03, 4.46it/s, lr: 7.0e-05 loss: 1.599e-01] jam: 2%|▏ | 469/20000 [02:18<1:08:11, 4.77it/s, lr: 7.0e-05 loss: 1.599e-01] jam: 2%|▏ | 469/20000 [02:19<1:08:11, 4.77it/s, lr: 7.0e-05 loss: 1.751e-01] jam: 2%|▏ | 470/20000 [02:19<1:45:04, 3.10it/s, lr: 7.0e-05 loss: 1.751e-01] jam: 2%|▏ | 470/20000 [02:19<1:45:04, 3.10it/s, lr: 7.0e-05 loss: 1.557e-01] jam: 2%|▏ | 471/20000 [02:19<1:49:11, 2.98it/s, lr: 7.0e-05 loss: 1.557e-01] jam: 2%|▏ | 471/20000 [02:19<1:49:11, 2.98it/s, lr: 7.0e-05 loss: 1.744e-01] jam: 2%|▏ | 472/20000 [02:19<1:22:06, 3.96it/s, lr: 7.0e-05 loss: 1.744e-01] jam: 2%|▏ | 472/20000 [02:19<1:22:06, 3.96it/s, lr: 7.0e-05 loss: 8.167e-02] jam: 2%|▏ | 473/20000 [02:19<1:08:53, 4.72it/s, lr: 7.0e-05 loss: 8.167e-02] jam: 2%|▏ | 473/20000 [02:20<1:08:53, 4.72it/s, lr: 7.0e-05 loss: 1.369e-01] jam: 2%|▏ | 474/20000 [02:20<1:10:01, 4.65it/s, lr: 7.0e-05 loss: 1.369e-01] jam: 2%|▏ | 474/20000 [02:20<1:10:01, 4.65it/s, lr: 7.0e-05 loss: 5.246e-02] jam: 2%|▏ | 475/20000 [02:20<1:41:37, 3.20it/s, lr: 7.0e-05 loss: 5.246e-02] jam: 2%|▏ | 475/20000 [02:20<1:41:37, 3.20it/s, lr: 7.0e-05 loss: 1.704e-01] | |
| Saving at step 475 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000475.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 2%|▏ | 476/20000 [02:21<1:41:37, 3.20it/s, lr: 7.0e-05 loss: 1.174e-01] jam: 2%|▏ | 477/20000 [02:21<1:23:35, 3.89it/s, lr: 7.0e-05 loss: 1.174e-01] jam: 2%|▏ | 477/20000 [02:21<1:23:35, 3.89it/s, lr: 7.0e-05 loss: 3.402e-02] jam: 2%|▏ | 478/20000 [02:21<1:15:48, 4.29it/s, lr: 7.0e-05 loss: 3.402e-02] jam: 2%|▏ | 478/20000 [02:21<1:15:48, 4.29it/s, lr: 7.0e-05 loss: 1.275e-01] jam: 2%|▏ | 479/20000 [02:21<1:09:27, 4.68it/s, lr: 7.0e-05 loss: 1.275e-01] jam: 2%|▏ | 479/20000 [02:22<1:09:27, 4.68it/s, lr: 7.0e-05 loss: 1.153e-01] jam: 2%|▏ | 480/20000 [02:22<1:08:41, 4.74it/s, lr: 7.0e-05 loss: 1.153e-01] jam: 2%|▏ | 480/20000 [02:22<1:08:41, 4.74it/s, lr: 7.0e-05 loss: 1.440e-01] jam: 2%|▏ | 481/20000 [02:22<1:38:25, 3.31it/s, lr: 7.0e-05 loss: 1.440e-01] jam: 2%|▏ | 481/20000 [02:22<1:38:25, 3.31it/s, lr: 7.0e-05 loss: 2.071e-01] jam: 2%|▏ | 482/20000 [02:22<1:17:33, 4.19it/s, lr: 7.0e-05 loss: 2.071e-01] jam: 2%|▏ | 482/20000 [02:22<1:17:33, 4.19it/s, lr: 7.0e-05 loss: 2.143e-01] jam: 2%|▏ | 483/20000 [02:22<1:40:48, 3.23it/s, lr: 7.0e-05 loss: 2.143e-01] jam: 2%|▏ | 483/20000 [02:23<1:40:48, 3.23it/s, lr: 7.0e-05 loss: 1.243e-01] jam: 2%|▏ | 484/20000 [02:23<1:49:25, 2.97it/s, lr: 7.0e-05 loss: 1.243e-01] jam: 2%|▏ | 484/20000 [02:23<1:49:25, 2.97it/s, lr: 7.0e-05 loss: 1.150e-01] jam: 2%|▏ | 485/20000 [02:23<1:51:00, 2.93it/s, lr: 7.0e-05 loss: 1.150e-01] jam: 2%|▏ | 485/20000 [02:24<1:51:00, 2.93it/s, lr: 7.0e-05 loss: 1.596e-01] jam: 2%|▏ | 486/20000 [02:24<1:54:30, 2.84it/s, lr: 7.0e-05 loss: 1.596e-01] jam: 2%|▏ | 486/20000 [02:24<1:54:30, 2.84it/s, lr: 7.0e-05 loss: 1.452e-01] jam: 2%|▏ | 487/20000 [02:24<1:57:02, 2.78it/s, lr: 7.0e-05 loss: 1.452e-01] jam: 2%|▏ | 487/20000 [02:24<1:57:02, 2.78it/s, lr: 7.0e-05 loss: 1.621e-01] jam: 2%|▏ | 488/20000 [02:24<1:52:33, 2.89it/s, lr: 7.0e-05 loss: 1.621e-01] jam: 2%|▏ | 488/20000 [02:24<1:52:33, 2.89it/s, lr: 7.0e-05 loss: 2.402e-01] jam: 2%|▏ | 489/20000 [02:24<1:20:50, 4.02it/s, lr: 7.0e-05 loss: 2.402e-01] jam: 2%|▏ | 489/20000 [02:25<1:20:50, 4.02it/s, lr: 7.0e-05 loss: 2.948e-01] jam: 2%|▏ | 490/20000 [02:25<1:47:43, 3.02it/s, lr: 7.0e-05 loss: 2.948e-01] jam: 2%|▏ | 490/20000 [02:25<1:47:43, 3.02it/s, lr: 7.0e-05 loss: 1.282e-01] jam: 2%|▏ | 491/20000 [02:25<1:15:28, 4.31it/s, lr: 7.0e-05 loss: 1.282e-01] jam: 2%|▏ | 491/20000 [02:25<1:15:28, 4.31it/s, lr: 7.0e-05 loss: 1.815e-01] jam: 2%|▏ | 492/20000 [02:25<1:00:22, 5.39it/s, lr: 7.0e-05 loss: 1.815e-01] jam: 2%|▏ | 492/20000 [02:26<1:00:22, 5.39it/s, lr: 7.0e-05 loss: 2.383e-02] jam: 2%|▏ | 493/20000 [02:26<1:35:31, 3.40it/s, lr: 7.0e-05 loss: 2.383e-02] jam: 2%|▏ | 493/20000 [02:26<1:35:31, 3.40it/s, lr: 7.0e-05 loss: 1.910e-01] jam: 2%|▏ | 494/20000 [02:26<1:45:52, 3.07it/s, lr: 7.0e-05 loss: 1.910e-01] jam: 2%|▏ | 494/20000 [02:26<1:45:52, 3.07it/s, lr: 7.0e-05 loss: 2.058e-01] jam: 2%|▏ | 495/20000 [02:26<1:21:38, 3.98it/s, lr: 7.0e-05 loss: 2.058e-01] jam: 2%|▏ | 495/20000 [02:26<1:21:38, 3.98it/s, lr: 7.0e-05 loss: 1.726e-01] jam: 2%|▏ | 496/20000 [02:26<1:51:41, 2.91it/s, lr: 7.0e-05 loss: 1.726e-01] jam: 2%|▏ | 496/20000 [02:27<1:51:41, 2.91it/s, lr: 7.0e-05 loss: 1.610e-01] jam: 2%|▏ | 497/20000 [02:27<1:23:21, 3.90it/s, lr: 7.0e-05 loss: 1.610e-01] jam: 2%|▏ | 497/20000 [02:27<1:23:21, 3.90it/s, lr: 7.0e-05 loss: 1.568e-01] jam: 2%|▏ | 498/20000 [02:27<1:19:08, 4.11it/s, lr: 7.0e-05 loss: 1.568e-01] jam: 2%|▏ | 498/20000 [02:27<1:19:08, 4.11it/s, lr: 7.0e-05 loss: 1.220e-01] jam: 2%|▏ | 499/20000 [02:27<1:02:35, 5.19it/s, lr: 7.0e-05 loss: 1.220e-01] jam: 2%|▏ | 499/20000 [02:27<1:02:35, 5.19it/s, lr: 7.0e-05 loss: 2.876e-01] jam: 2%|▎ | 500/20000 [02:27<1:48:02, 3.01it/s, lr: 7.0e-05 loss: 2.876e-01] jam: 2%|▎ | 500/20000 [02:28<1:48:02, 3.01it/s, lr: 7.0e-05 loss: 8.951e-02] | |
| Saving at step 500 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000500.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 501/20000 [02:29<1:48:01, 3.01it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 3%|▎ | 502/20000 [02:29<1:34:04, 3.45it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 3%|▎ | 502/20000 [02:29<1:34:04, 3.45it/s, lr: 7.0e-05 loss: 2.293e-01] jam: 3%|▎ | 503/20000 [02:29<1:43:14, 3.15it/s, lr: 7.0e-05 loss: 2.293e-01] jam: 3%|▎ | 503/20000 [02:29<1:43:14, 3.15it/s, lr: 7.0e-05 loss: 1.391e-01] jam: 3%|▎ | 504/20000 [02:29<1:55:07, 2.82it/s, lr: 7.0e-05 loss: 1.391e-01] jam: 3%|▎ | 504/20000 [02:29<1:55:07, 2.82it/s, lr: 7.0e-05 loss: 1.560e-01] jam: 3%|▎ | 505/20000 [02:29<1:15:11, 4.32it/s, lr: 7.0e-05 loss: 1.560e-01] jam: 3%|▎ | 505/20000 [02:30<1:15:11, 4.32it/s, lr: 7.0e-05 loss: 2.068e-01] jam: 3%|▎ | 506/20000 [02:30<1:42:00, 3.19it/s, lr: 7.0e-05 loss: 2.068e-01] jam: 3%|▎ | 506/20000 [02:30<1:42:00, 3.19it/s, lr: 7.0e-05 loss: 1.321e-01] jam: 3%|▎ | 507/20000 [02:30<1:55:03, 2.82it/s, lr: 7.0e-05 loss: 1.321e-01] jam: 3%|▎ | 507/20000 [02:31<1:55:03, 2.82it/s, lr: 7.0e-05 loss: 1.415e-01] jam: 3%|▎ | 508/20000 [02:31<1:52:34, 2.89it/s, lr: 7.0e-05 loss: 1.415e-01] jam: 3%|▎ | 508/20000 [02:31<1:52:34, 2.89it/s, lr: 7.0e-05 loss: 1.867e-01] jam: 3%|▎ | 509/20000 [02:31<1:54:12, 2.84it/s, lr: 7.0e-05 loss: 1.867e-01] jam: 3%|▎ | 509/20000 [02:31<1:54:12, 2.84it/s, lr: 7.0e-05 loss: 1.260e-01] jam: 3%|▎ | 510/20000 [02:31<1:52:42, 2.88it/s, lr: 7.0e-05 loss: 1.260e-01] jam: 3%|▎ | 510/20000 [02:31<1:52:42, 2.88it/s, lr: 7.0e-05 loss: 1.524e-01] jam: 3%|▎ | 511/20000 [02:31<1:22:20, 3.94it/s, lr: 7.0e-05 loss: 1.524e-01] jam: 3%|▎ | 511/20000 [02:32<1:22:20, 3.94it/s, lr: 7.0e-05 loss: 8.011e-02] jam: 3%|▎ | 512/20000 [02:32<1:50:02, 2.95it/s, lr: 7.0e-05 loss: 8.011e-02] jam: 3%|▎ | 512/20000 [02:32<1:50:02, 2.95it/s, lr: 7.0e-05 loss: 7.163e-02] jam: 3%|▎ | 513/20000 [02:32<1:56:48, 2.78it/s, lr: 7.0e-05 loss: 7.163e-02] jam: 3%|▎ | 513/20000 [02:32<1:56:48, 2.78it/s, lr: 7.0e-05 loss: 9.148e-02] jam: 3%|▎ | 514/20000 [02:32<1:17:50, 4.17it/s, lr: 7.0e-05 loss: 9.148e-02] jam: 3%|▎ | 514/20000 [02:33<1:17:50, 4.17it/s, lr: 7.0e-05 loss: 9.592e-02] jam: 3%|▎ | 515/20000 [02:33<1:40:53, 3.22it/s, lr: 7.0e-05 loss: 9.592e-02] jam: 3%|▎ | 515/20000 [02:33<1:40:53, 3.22it/s, lr: 7.0e-05 loss: 8.666e-02] jam: 3%|▎ | 516/20000 [02:33<1:53:03, 2.87it/s, lr: 7.0e-05 loss: 8.666e-02] jam: 3%|▎ | 516/20000 [02:33<1:53:03, 2.87it/s, lr: 7.0e-05 loss: 9.021e-02] jam: 3%|▎ | 517/20000 [02:33<1:51:14, 2.92it/s, lr: 7.0e-05 loss: 9.021e-02] jam: 3%|▎ | 517/20000 [02:34<1:51:14, 2.92it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 3%|▎ | 518/20000 [02:34<1:23:12, 3.90it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 3%|▎ | 518/20000 [02:34<1:23:12, 3.90it/s, lr: 7.0e-05 loss: 2.192e-01] jam: 3%|▎ | 519/20000 [02:34<1:49:37, 2.96it/s, lr: 7.0e-05 loss: 2.192e-01] jam: 3%|▎ | 519/20000 [02:34<1:49:37, 2.96it/s, lr: 7.0e-05 loss: 1.185e-01] jam: 3%|▎ | 520/20000 [02:34<1:24:20, 3.85it/s, lr: 7.0e-05 loss: 1.185e-01] jam: 3%|▎ | 520/20000 [02:34<1:24:20, 3.85it/s, lr: 7.0e-05 loss: 1.373e-01] jam: 3%|▎ | 521/20000 [02:34<1:27:24, 3.71it/s, lr: 7.0e-05 loss: 1.373e-01] jam: 3%|▎ | 521/20000 [02:35<1:27:24, 3.71it/s, lr: 7.0e-05 loss: 1.480e-01] jam: 3%|▎ | 522/20000 [02:35<1:19:56, 4.06it/s, lr: 7.0e-05 loss: 1.480e-01] jam: 3%|▎ | 522/20000 [02:35<1:19:56, 4.06it/s, lr: 7.0e-05 loss: 1.142e-01] jam: 3%|▎ | 523/20000 [02:35<1:46:32, 3.05it/s, lr: 7.0e-05 loss: 1.142e-01] jam: 3%|▎ | 523/20000 [02:35<1:46:32, 3.05it/s, lr: 7.0e-05 loss: 4.873e-02] jam: 3%|▎ | 524/20000 [02:35<1:56:53, 2.78it/s, lr: 7.0e-05 loss: 4.873e-02] jam: 3%|▎ | 524/20000 [02:36<1:56:53, 2.78it/s, lr: 7.0e-05 loss: 2.195e-01] jam: 3%|▎ | 525/20000 [02:36<1:52:17, 2.89it/s, lr: 7.0e-05 loss: 2.195e-01] jam: 3%|▎ | 525/20000 [02:36<1:52:17, 2.89it/s, lr: 7.0e-05 loss: 2.510e-01] | |
| Saving at step 525 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000525.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 526/20000 [02:37<1:52:17, 2.89it/s, lr: 7.0e-05 loss: 2.827e-01] jam: 3%|▎ | 527/20000 [02:37<1:27:53, 3.69it/s, lr: 7.0e-05 loss: 2.827e-01] jam: 3%|▎ | 527/20000 [02:37<1:27:53, 3.69it/s, lr: 7.0e-05 loss: 2.249e-01] jam: 3%|▎ | 528/20000 [02:37<1:11:08, 4.56it/s, lr: 7.0e-05 loss: 2.249e-01] jam: 3%|▎ | 528/20000 [02:37<1:11:08, 4.56it/s, lr: 7.0e-05 loss: 2.496e-01] jam: 3%|▎ | 529/20000 [02:37<1:06:26, 4.88it/s, lr: 7.0e-05 loss: 2.496e-01] jam: 3%|▎ | 529/20000 [02:38<1:06:26, 4.88it/s, lr: 7.0e-05 loss: 1.973e-01] jam: 3%|▎ | 530/20000 [02:38<1:38:17, 3.30it/s, lr: 7.0e-05 loss: 1.973e-01] jam: 3%|▎ | 530/20000 [02:38<1:38:17, 3.30it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 3%|▎ | 531/20000 [02:38<1:54:32, 2.83it/s, lr: 7.0e-05 loss: 1.300e-01] jam: 3%|▎ | 531/20000 [02:38<1:54:32, 2.83it/s, lr: 7.0e-05 loss: 9.370e-02] jam: 3%|▎ | 532/20000 [02:38<1:51:47, 2.90it/s, lr: 7.0e-05 loss: 9.370e-02] jam: 3%|▎ | 532/20000 [02:39<1:51:47, 2.90it/s, lr: 7.0e-05 loss: 2.402e-01] jam: 3%|▎ | 533/20000 [02:39<1:23:17, 3.90it/s, lr: 7.0e-05 loss: 2.402e-01] jam: 3%|▎ | 533/20000 [02:39<1:23:17, 3.90it/s, lr: 7.0e-05 loss: 1.190e-01] jam: 3%|▎ | 534/20000 [02:39<1:48:05, 3.00it/s, lr: 7.0e-05 loss: 1.190e-01] jam: 3%|▎ | 534/20000 [02:39<1:48:05, 3.00it/s, lr: 7.0e-05 loss: 9.711e-02] jam: 3%|▎ | 535/20000 [02:39<1:57:21, 2.76it/s, lr: 7.0e-05 loss: 9.711e-02] jam: 3%|▎ | 535/20000 [02:40<1:57:21, 2.76it/s, lr: 7.0e-05 loss: 3.042e-01] jam: 3%|▎ | 536/20000 [02:40<1:18:07, 4.15it/s, lr: 7.0e-05 loss: 3.042e-01] jam: 3%|▎ | 536/20000 [02:40<1:18:07, 4.15it/s, lr: 7.0e-05 loss: 9.257e-02] jam: 3%|▎ | 537/20000 [02:40<1:05:35, 4.95it/s, lr: 7.0e-05 loss: 9.257e-02] jam: 3%|▎ | 537/20000 [02:40<1:05:35, 4.95it/s, lr: 7.0e-05 loss: 2.003e-01] jam: 3%|▎ | 538/20000 [02:40<1:39:12, 3.27it/s, lr: 7.0e-05 loss: 2.003e-01] jam: 3%|▎ | 538/20000 [02:40<1:39:12, 3.27it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 3%|▎ | 539/20000 [02:40<1:55:22, 2.81it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 3%|▎ | 539/20000 [02:41<1:55:22, 2.81it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 3%|▎ | 540/20000 [02:41<1:17:10, 4.20it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 3%|▎ | 540/20000 [02:41<1:17:10, 4.20it/s, lr: 7.0e-05 loss: 2.119e-01] jam: 3%|▎ | 541/20000 [02:41<1:06:48, 4.85it/s, lr: 7.0e-05 loss: 2.119e-01] jam: 3%|▎ | 541/20000 [02:41<1:06:48, 4.85it/s, lr: 7.0e-05 loss: 3.128e-01] jam: 3%|▎ | 542/20000 [02:41<1:06:59, 4.84it/s, lr: 7.0e-05 loss: 3.128e-01] jam: 3%|▎ | 542/20000 [02:41<1:06:59, 4.84it/s, lr: 7.0e-05 loss: 7.539e-02] jam: 3%|▎ | 543/20000 [02:41<1:03:04, 5.14it/s, lr: 7.0e-05 loss: 7.539e-02] jam: 3%|▎ | 543/20000 [02:41<1:03:04, 5.14it/s, lr: 7.0e-05 loss: 1.649e-01] jam: 3%|▎ | 544/20000 [02:41<1:05:39, 4.94it/s, lr: 7.0e-05 loss: 1.649e-01] jam: 3%|▎ | 544/20000 [02:42<1:05:39, 4.94it/s, lr: 7.0e-05 loss: 2.546e-01] jam: 3%|▎ | 545/20000 [02:42<1:11:38, 4.53it/s, lr: 7.0e-05 loss: 2.546e-01] jam: 3%|▎ | 545/20000 [02:42<1:11:38, 4.53it/s, lr: 7.0e-05 loss: 1.565e-01] jam: 3%|▎ | 546/20000 [02:42<1:38:47, 3.28it/s, lr: 7.0e-05 loss: 1.565e-01] jam: 3%|▎ | 546/20000 [02:42<1:38:47, 3.28it/s, lr: 7.0e-05 loss: 2.572e-01] jam: 3%|▎ | 547/20000 [02:42<1:19:22, 4.08it/s, lr: 7.0e-05 loss: 2.572e-01] jam: 3%|▎ | 547/20000 [02:42<1:19:22, 4.08it/s, lr: 7.0e-05 loss: 5.197e-02] jam: 3%|▎ | 548/20000 [02:42<1:11:18, 4.55it/s, lr: 7.0e-05 loss: 5.197e-02] jam: 3%|▎ | 548/20000 [02:43<1:11:18, 4.55it/s, lr: 7.0e-05 loss: 8.758e-02] jam: 3%|▎ | 549/20000 [02:43<1:04:17, 5.04it/s, lr: 7.0e-05 loss: 8.758e-02] jam: 3%|▎ | 549/20000 [02:43<1:04:17, 5.04it/s, lr: 7.0e-05 loss: 1.149e-01] jam: 3%|▎ | 550/20000 [02:43<1:03:42, 5.09it/s, lr: 7.0e-05 loss: 1.149e-01] jam: 3%|▎ | 550/20000 [02:43<1:03:42, 5.09it/s, lr: 7.0e-05 loss: 2.407e-01] | |
| Saving at step 550 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000550.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 551/20000 [02:44<1:03:42, 5.09it/s, lr: 7.0e-05 loss: 2.000e-01] jam: 3%|▎ | 552/20000 [02:44<1:18:22, 4.14it/s, lr: 7.0e-05 loss: 2.000e-01] jam: 3%|▎ | 552/20000 [02:44<1:18:22, 4.14it/s, lr: 7.0e-05 loss: 1.947e-01] jam: 3%|▎ | 553/20000 [02:44<1:05:15, 4.97it/s, lr: 7.0e-05 loss: 1.947e-01] jam: 3%|▎ | 553/20000 [02:44<1:05:15, 4.97it/s, lr: 7.0e-05 loss: 8.454e-02] jam: 3%|▎ | 554/20000 [02:44<1:35:13, 3.40it/s, lr: 7.0e-05 loss: 8.454e-02] jam: 3%|▎ | 554/20000 [02:45<1:35:13, 3.40it/s, lr: 7.0e-05 loss: 1.503e-01] jam: 3%|▎ | 555/20000 [02:45<1:06:07, 4.90it/s, lr: 7.0e-05 loss: 1.503e-01] jam: 3%|▎ | 555/20000 [02:45<1:06:07, 4.90it/s, lr: 7.0e-05 loss: 6.752e-02] jam: 3%|▎ | 556/20000 [02:45<1:41:01, 3.21it/s, lr: 7.0e-05 loss: 6.752e-02] jam: 3%|▎ | 556/20000 [02:45<1:41:01, 3.21it/s, lr: 7.0e-05 loss: 1.258e-01] jam: 3%|▎ | 557/20000 [02:45<1:55:03, 2.82it/s, lr: 7.0e-05 loss: 1.258e-01] jam: 3%|▎ | 557/20000 [02:45<1:55:03, 2.82it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 3%|▎ | 558/20000 [02:45<1:16:18, 4.25it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 3%|▎ | 558/20000 [02:46<1:16:18, 4.25it/s, lr: 7.0e-05 loss: 2.163e-01] jam: 3%|▎ | 559/20000 [02:46<1:40:13, 3.23it/s, lr: 7.0e-05 loss: 2.163e-01] jam: 3%|▎ | 559/20000 [02:46<1:40:13, 3.23it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 3%|▎ | 560/20000 [02:46<1:55:13, 2.81it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 3%|▎ | 560/20000 [02:46<1:55:13, 2.81it/s, lr: 7.0e-05 loss: 1.595e-01] jam: 3%|▎ | 561/20000 [02:46<1:17:04, 4.20it/s, lr: 7.0e-05 loss: 1.595e-01] jam: 3%|▎ | 561/20000 [02:47<1:17:04, 4.20it/s, lr: 7.0e-05 loss: 1.234e-01] jam: 3%|▎ | 562/20000 [02:47<1:15:46, 4.28it/s, lr: 7.0e-05 loss: 1.234e-01] jam: 3%|▎ | 562/20000 [02:47<1:15:46, 4.28it/s, lr: 7.0e-05 loss: 7.366e-02] jam: 3%|▎ | 563/20000 [02:47<1:39:57, 3.24it/s, lr: 7.0e-05 loss: 7.366e-02] jam: 3%|▎ | 563/20000 [02:47<1:39:57, 3.24it/s, lr: 7.0e-05 loss: 5.388e-02] jam: 3%|▎ | 564/20000 [02:47<1:51:29, 2.91it/s, lr: 7.0e-05 loss: 5.388e-02] jam: 3%|▎ | 564/20000 [02:47<1:51:29, 2.91it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 3%|▎ | 565/20000 [02:47<1:21:01, 4.00it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 3%|▎ | 565/20000 [02:48<1:21:01, 4.00it/s, lr: 7.0e-05 loss: 3.209e-02] jam: 3%|▎ | 566/20000 [02:48<1:48:43, 2.98it/s, lr: 7.0e-05 loss: 3.209e-02] jam: 3%|▎ | 566/20000 [02:48<1:48:43, 2.98it/s, lr: 7.0e-05 loss: 2.234e-01] jam: 3%|▎ | 567/20000 [02:48<1:19:17, 4.08it/s, lr: 7.0e-05 loss: 2.234e-01] jam: 3%|▎ | 567/20000 [02:48<1:19:17, 4.08it/s, lr: 7.0e-05 loss: 8.231e-02] jam: 3%|▎ | 568/20000 [02:48<1:00:47, 5.33it/s, lr: 7.0e-05 loss: 8.231e-02] jam: 3%|▎ | 568/20000 [02:48<1:00:47, 5.33it/s, lr: 7.0e-05 loss: 2.344e-01] jam: 3%|▎ | 569/20000 [02:48<1:02:45, 5.16it/s, lr: 7.0e-05 loss: 2.344e-01] jam: 3%|▎ | 569/20000 [02:49<1:02:45, 5.16it/s, lr: 7.0e-05 loss: 1.393e-01] jam: 3%|▎ | 570/20000 [02:49<1:11:39, 4.52it/s, lr: 7.0e-05 loss: 1.393e-01] jam: 3%|▎ | 570/20000 [02:49<1:11:39, 4.52it/s, lr: 7.0e-05 loss: 2.364e-01] jam: 3%|▎ | 571/20000 [02:49<1:06:21, 4.88it/s, lr: 7.0e-05 loss: 2.364e-01] jam: 3%|▎ | 571/20000 [02:49<1:06:21, 4.88it/s, lr: 7.0e-05 loss: 2.693e-01] jam: 3%|▎ | 572/20000 [02:49<1:03:12, 5.12it/s, lr: 7.0e-05 loss: 2.693e-01] jam: 3%|▎ | 572/20000 [02:49<1:03:12, 5.12it/s, lr: 7.0e-05 loss: 1.049e-01] jam: 3%|▎ | 573/20000 [02:49<1:04:55, 4.99it/s, lr: 7.0e-05 loss: 1.049e-01] jam: 3%|▎ | 573/20000 [02:49<1:04:55, 4.99it/s, lr: 7.0e-05 loss: 1.285e-01] jam: 3%|▎ | 574/20000 [02:49<1:07:35, 4.79it/s, lr: 7.0e-05 loss: 1.285e-01] jam: 3%|▎ | 574/20000 [02:50<1:07:35, 4.79it/s, lr: 7.0e-05 loss: 7.886e-02] jam: 3%|▎ | 575/20000 [02:50<1:01:52, 5.23it/s, lr: 7.0e-05 loss: 7.886e-02] jam: 3%|▎ | 575/20000 [02:50<1:01:52, 5.23it/s, lr: 7.0e-05 loss: 8.077e-02] | |
| Saving at step 575 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000575.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 576/20000 [02:51<1:01:52, 5.23it/s, lr: 7.0e-05 loss: 2.995e-01] jam: 3%|▎ | 577/20000 [02:51<1:14:39, 4.34it/s, lr: 7.0e-05 loss: 2.995e-01] jam: 3%|▎ | 577/20000 [02:51<1:14:39, 4.34it/s, lr: 7.0e-05 loss: 1.735e-01] jam: 3%|▎ | 578/20000 [02:51<1:14:58, 4.32it/s, lr: 7.0e-05 loss: 1.735e-01] jam: 3%|▎ | 578/20000 [02:51<1:14:58, 4.32it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 3%|▎ | 579/20000 [02:51<1:05:11, 4.96it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 3%|▎ | 579/20000 [02:51<1:05:11, 4.96it/s, lr: 7.0e-05 loss: 1.981e-01] jam: 3%|▎ | 580/20000 [02:51<1:39:14, 3.26it/s, lr: 7.0e-05 loss: 1.981e-01] jam: 3%|▎ | 580/20000 [02:52<1:39:14, 3.26it/s, lr: 7.0e-05 loss: 1.048e-01] jam: 3%|▎ | 581/20000 [02:52<1:19:01, 4.10it/s, lr: 7.0e-05 loss: 1.048e-01] jam: 3%|▎ | 581/20000 [02:52<1:19:01, 4.10it/s, lr: 7.0e-05 loss: 1.488e-01] jam: 3%|▎ | 582/20000 [02:52<1:45:58, 3.05it/s, lr: 7.0e-05 loss: 1.488e-01] jam: 3%|▎ | 582/20000 [02:52<1:45:58, 3.05it/s, lr: 7.0e-05 loss: 2.326e-01] jam: 3%|▎ | 583/20000 [02:52<1:48:42, 2.98it/s, lr: 7.0e-05 loss: 2.326e-01] jam: 3%|▎ | 583/20000 [02:53<1:48:42, 2.98it/s, lr: 7.0e-05 loss: 5.856e-02] jam: 3%|▎ | 584/20000 [02:53<1:50:33, 2.93it/s, lr: 7.0e-05 loss: 5.856e-02] jam: 3%|▎ | 584/20000 [02:53<1:50:33, 2.93it/s, lr: 7.0e-05 loss: 1.394e-01] jam: 3%|▎ | 585/20000 [02:53<1:18:02, 4.15it/s, lr: 7.0e-05 loss: 1.394e-01] jam: 3%|▎ | 585/20000 [02:53<1:18:02, 4.15it/s, lr: 7.0e-05 loss: 1.720e-01] jam: 3%|▎ | 586/20000 [02:53<1:41:26, 3.19it/s, lr: 7.0e-05 loss: 1.720e-01] jam: 3%|▎ | 586/20000 [02:54<1:41:26, 3.19it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 3%|▎ | 587/20000 [02:54<1:55:13, 2.81it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 3%|▎ | 587/20000 [02:54<1:55:13, 2.81it/s, lr: 7.0e-05 loss: 7.807e-02] jam: 3%|▎ | 588/20000 [02:54<1:19:13, 4.08it/s, lr: 7.0e-05 loss: 7.807e-02] jam: 3%|▎ | 588/20000 [02:54<1:19:13, 4.08it/s, lr: 7.0e-05 loss: 6.330e-02] jam: 3%|▎ | 589/20000 [02:54<1:06:00, 4.90it/s, lr: 7.0e-05 loss: 6.330e-02] jam: 3%|▎ | 589/20000 [02:54<1:06:00, 4.90it/s, lr: 7.0e-05 loss: 7.166e-02] jam: 3%|▎ | 590/20000 [02:54<1:37:10, 3.33it/s, lr: 7.0e-05 loss: 7.166e-02] jam: 3%|▎ | 590/20000 [02:55<1:37:10, 3.33it/s, lr: 7.0e-05 loss: 1.722e-01] jam: 3%|▎ | 591/20000 [02:55<1:53:36, 2.85it/s, lr: 7.0e-05 loss: 1.722e-01] jam: 3%|▎ | 591/20000 [02:55<1:53:36, 2.85it/s, lr: 7.0e-05 loss: 4.103e-02] jam: 3%|▎ | 592/20000 [02:55<1:17:27, 4.18it/s, lr: 7.0e-05 loss: 4.103e-02] jam: 3%|▎ | 592/20000 [02:55<1:17:27, 4.18it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 3%|▎ | 593/20000 [02:55<1:08:16, 4.74it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 3%|▎ | 593/20000 [02:55<1:08:16, 4.74it/s, lr: 7.0e-05 loss: 1.801e-01] jam: 3%|▎ | 594/20000 [02:55<1:04:58, 4.98it/s, lr: 7.0e-05 loss: 1.801e-01] jam: 3%|▎ | 594/20000 [02:56<1:04:58, 4.98it/s, lr: 7.0e-05 loss: 7.540e-02] jam: 3%|▎ | 595/20000 [02:56<1:36:34, 3.35it/s, lr: 7.0e-05 loss: 7.540e-02] jam: 3%|▎ | 595/20000 [02:56<1:36:34, 3.35it/s, lr: 7.0e-05 loss: 1.521e-01] jam: 3%|▎ | 596/20000 [02:56<1:53:32, 2.85it/s, lr: 7.0e-05 loss: 1.521e-01] jam: 3%|▎ | 596/20000 [02:56<1:53:32, 2.85it/s, lr: 7.0e-05 loss: 1.022e-01] jam: 3%|▎ | 597/20000 [02:56<1:49:43, 2.95it/s, lr: 7.0e-05 loss: 1.022e-01] jam: 3%|▎ | 597/20000 [02:57<1:49:43, 2.95it/s, lr: 7.0e-05 loss: 1.174e-01] jam: 3%|▎ | 598/20000 [02:57<1:55:52, 2.79it/s, lr: 7.0e-05 loss: 1.174e-01] jam: 3%|▎ | 598/20000 [02:57<1:55:52, 2.79it/s, lr: 7.0e-05 loss: 1.980e-02] jam: 3%|▎ | 599/20000 [02:57<1:51:51, 2.89it/s, lr: 7.0e-05 loss: 1.980e-02] jam: 3%|▎ | 599/20000 [02:57<1:51:51, 2.89it/s, lr: 7.0e-05 loss: 8.669e-02] jam: 3%|▎ | 600/20000 [02:57<1:54:40, 2.82it/s, lr: 7.0e-05 loss: 8.669e-02] jam: 3%|▎ | 600/20000 [02:58<1:54:40, 2.82it/s, lr: 7.0e-05 loss: 1.922e-01] | |
| Saving at step 600 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000600.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 601/20000 [02:58<1:54:40, 2.82it/s, lr: 7.0e-05 loss: 1.909e-01] jam: 3%|▎ | 602/20000 [02:58<1:25:09, 3.80it/s, lr: 7.0e-05 loss: 1.909e-01] jam: 3%|▎ | 602/20000 [02:58<1:25:09, 3.80it/s, lr: 7.0e-05 loss: 1.467e-01] jam: 3%|▎ | 603/20000 [02:58<1:43:26, 3.13it/s, lr: 7.0e-05 loss: 1.467e-01] jam: 3%|▎ | 603/20000 [02:59<1:43:26, 3.13it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 3%|▎ | 604/20000 [02:59<1:11:24, 4.53it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 3%|▎ | 604/20000 [02:59<1:11:24, 4.53it/s, lr: 7.0e-05 loss: 1.146e-01] jam: 3%|▎ | 605/20000 [02:59<1:00:09, 5.37it/s, lr: 7.0e-05 loss: 1.146e-01] jam: 3%|▎ | 605/20000 [02:59<1:00:09, 5.37it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 3%|▎ | 606/20000 [02:59<1:39:19, 3.25it/s, lr: 7.0e-05 loss: 1.476e-01] jam: 3%|▎ | 606/20000 [02:59<1:39:19, 3.25it/s, lr: 7.0e-05 loss: 6.734e-02] jam: 3%|▎ | 607/20000 [02:59<1:46:23, 3.04it/s, lr: 7.0e-05 loss: 6.734e-02] jam: 3%|▎ | 607/20000 [03:00<1:46:23, 3.04it/s, lr: 7.0e-05 loss: 1.014e-01] jam: 3%|▎ | 608/20000 [03:00<1:33:54, 3.44it/s, lr: 7.0e-05 loss: 1.014e-01] jam: 3%|▎ | 608/20000 [03:00<1:33:54, 3.44it/s, lr: 7.0e-05 loss: 1.600e-01] jam: 3%|▎ | 609/20000 [03:00<1:12:21, 4.47it/s, lr: 7.0e-05 loss: 1.600e-01] jam: 3%|▎ | 609/20000 [03:00<1:12:21, 4.47it/s, lr: 7.0e-05 loss: 2.003e-01] jam: 3%|▎ | 610/20000 [03:00<58:35, 5.52it/s, lr: 7.0e-05 loss: 2.003e-01] jam: 3%|▎ | 610/20000 [03:00<58:35, 5.52it/s, lr: 7.0e-05 loss: 1.645e-01] jam: 3%|▎ | 611/20000 [03:00<1:34:53, 3.41it/s, lr: 7.0e-05 loss: 1.645e-01] jam: 3%|▎ | 611/20000 [03:01<1:34:53, 3.41it/s, lr: 7.0e-05 loss: 1.785e-01] jam: 3%|▎ | 612/20000 [03:01<1:19:37, 4.06it/s, lr: 7.0e-05 loss: 1.785e-01] jam: 3%|▎ | 612/20000 [03:01<1:19:37, 4.06it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 3%|▎ | 613/20000 [03:01<1:14:03, 4.36it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 3%|▎ | 613/20000 [03:01<1:14:03, 4.36it/s, lr: 7.0e-05 loss: 2.527e-01] jam: 3%|▎ | 614/20000 [03:01<1:10:14, 4.60it/s, lr: 7.0e-05 loss: 2.527e-01] jam: 3%|▎ | 614/20000 [03:01<1:10:14, 4.60it/s, lr: 7.0e-05 loss: 1.941e-02] jam: 3%|▎ | 615/20000 [03:01<58:18, 5.54it/s, lr: 7.0e-05 loss: 1.941e-02] jam: 3%|▎ | 615/20000 [03:02<58:18, 5.54it/s, lr: 7.0e-05 loss: 2.219e-01] jam: 3%|▎ | 616/20000 [03:02<1:41:18, 3.19it/s, lr: 7.0e-05 loss: 2.219e-01] jam: 3%|▎ | 616/20000 [03:02<1:41:18, 3.19it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 3%|▎ | 617/20000 [03:02<1:07:33, 4.78it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 3%|▎ | 617/20000 [03:02<1:07:33, 4.78it/s, lr: 7.0e-05 loss: 1.785e-01] jam: 3%|▎ | 618/20000 [03:02<59:22, 5.44it/s, lr: 7.0e-05 loss: 1.785e-01] jam: 3%|▎ | 618/20000 [03:02<59:22, 5.44it/s, lr: 7.0e-05 loss: 2.639e-01] jam: 3%|▎ | 619/20000 [03:02<1:42:42, 3.14it/s, lr: 7.0e-05 loss: 2.639e-01] jam: 3%|▎ | 619/20000 [03:03<1:42:42, 3.14it/s, lr: 7.0e-05 loss: 2.813e-01] jam: 3%|▎ | 620/20000 [03:03<1:11:51, 4.50it/s, lr: 7.0e-05 loss: 2.813e-01] jam: 3%|▎ | 620/20000 [03:03<1:11:51, 4.50it/s, lr: 7.0e-05 loss: 1.640e-01] jam: 3%|▎ | 621/20000 [03:03<1:38:32, 3.28it/s, lr: 7.0e-05 loss: 1.640e-01] jam: 3%|▎ | 621/20000 [03:03<1:38:32, 3.28it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 3%|▎ | 622/20000 [03:03<1:50:51, 2.91it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 3%|▎ | 622/20000 [03:04<1:50:51, 2.91it/s, lr: 7.0e-05 loss: 1.152e-01] jam: 3%|▎ | 623/20000 [03:04<1:50:35, 2.92it/s, lr: 7.0e-05 loss: 1.152e-01] jam: 3%|▎ | 623/20000 [03:04<1:50:35, 2.92it/s, lr: 7.0e-05 loss: 5.402e-02] jam: 3%|▎ | 624/20000 [03:04<1:22:26, 3.92it/s, lr: 7.0e-05 loss: 5.402e-02] jam: 3%|▎ | 624/20000 [03:04<1:22:26, 3.92it/s, lr: 7.0e-05 loss: 2.607e-01] jam: 3%|▎ | 625/20000 [03:04<1:11:29, 4.52it/s, lr: 7.0e-05 loss: 2.607e-01] jam: 3%|▎ | 625/20000 [03:04<1:11:29, 4.52it/s, lr: 7.0e-05 loss: 3.251e-01] | |
| Saving at step 625 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000625.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 626/20000 [03:05<1:11:29, 4.52it/s, lr: 7.0e-05 loss: 5.536e-02] jam: 3%|▎ | 627/20000 [03:05<1:50:28, 2.92it/s, lr: 7.0e-05 loss: 5.536e-02] jam: 3%|▎ | 627/20000 [03:05<1:50:28, 2.92it/s, lr: 7.0e-05 loss: 1.717e-01] jam: 3%|▎ | 628/20000 [03:05<1:49:43, 2.94it/s, lr: 7.0e-05 loss: 1.717e-01] jam: 3%|▎ | 628/20000 [03:05<1:49:43, 2.94it/s, lr: 7.0e-05 loss: 6.585e-02] jam: 3%|▎ | 629/20000 [03:05<1:20:40, 4.00it/s, lr: 7.0e-05 loss: 6.585e-02] jam: 3%|▎ | 629/20000 [03:06<1:20:40, 4.00it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 3%|▎ | 630/20000 [03:06<1:06:37, 4.85it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 3%|▎ | 630/20000 [03:06<1:06:37, 4.85it/s, lr: 7.0e-05 loss: 2.377e-01] jam: 3%|▎ | 631/20000 [03:06<1:38:31, 3.28it/s, lr: 7.0e-05 loss: 2.377e-01] jam: 3%|▎ | 631/20000 [03:06<1:38:31, 3.28it/s, lr: 7.0e-05 loss: 1.389e-01] jam: 3%|▎ | 632/20000 [03:06<1:22:18, 3.92it/s, lr: 7.0e-05 loss: 1.389e-01] jam: 3%|▎ | 632/20000 [03:07<1:22:18, 3.92it/s, lr: 7.0e-05 loss: 1.081e-01] jam: 3%|▎ | 633/20000 [03:07<1:48:03, 2.99it/s, lr: 7.0e-05 loss: 1.081e-01] jam: 3%|▎ | 633/20000 [03:07<1:48:03, 2.99it/s, lr: 7.0e-05 loss: 1.490e-01] jam: 3%|▎ | 634/20000 [03:07<1:33:59, 3.43it/s, lr: 7.0e-05 loss: 1.490e-01] jam: 3%|▎ | 634/20000 [03:07<1:33:59, 3.43it/s, lr: 7.0e-05 loss: 2.143e-01] jam: 3%|▎ | 635/20000 [03:07<1:46:13, 3.04it/s, lr: 7.0e-05 loss: 2.143e-01] jam: 3%|▎ | 635/20000 [03:08<1:46:13, 3.04it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 3%|▎ | 636/20000 [03:08<1:49:03, 2.96it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 3%|▎ | 636/20000 [03:08<1:49:03, 2.96it/s, lr: 7.0e-05 loss: 1.453e-01] jam: 3%|▎ | 637/20000 [03:08<1:50:20, 2.92it/s, lr: 7.0e-05 loss: 1.453e-01] jam: 3%|▎ | 637/20000 [03:08<1:50:20, 2.92it/s, lr: 7.0e-05 loss: 7.063e-02] jam: 3%|▎ | 638/20000 [03:08<1:54:13, 2.83it/s, lr: 7.0e-05 loss: 7.063e-02] jam: 3%|▎ | 638/20000 [03:08<1:54:13, 2.83it/s, lr: 7.0e-05 loss: 7.632e-02] jam: 3%|▎ | 639/20000 [03:08<1:19:18, 4.07it/s, lr: 7.0e-05 loss: 7.632e-02] jam: 3%|▎ | 639/20000 [03:09<1:19:18, 4.07it/s, lr: 7.0e-05 loss: 1.549e-01] jam: 3%|▎ | 640/20000 [03:09<1:04:14, 5.02it/s, lr: 7.0e-05 loss: 1.549e-01] jam: 3%|▎ | 640/20000 [03:09<1:04:14, 5.02it/s, lr: 7.0e-05 loss: 5.184e-02] jam: 3%|▎ | 641/20000 [03:09<1:38:06, 3.29it/s, lr: 7.0e-05 loss: 5.184e-02] jam: 3%|▎ | 641/20000 [03:09<1:38:06, 3.29it/s, lr: 7.0e-05 loss: 3.633e-02] jam: 3%|▎ | 642/20000 [03:09<1:19:51, 4.04it/s, lr: 7.0e-05 loss: 3.633e-02] jam: 3%|▎ | 642/20000 [03:09<1:19:51, 4.04it/s, lr: 7.0e-05 loss: 5.224e-02] jam: 3%|▎ | 643/20000 [03:09<1:14:14, 4.35it/s, lr: 7.0e-05 loss: 5.224e-02] jam: 3%|▎ | 643/20000 [03:10<1:14:14, 4.35it/s, lr: 7.0e-05 loss: 1.719e-01] jam: 3%|▎ | 644/20000 [03:10<1:39:33, 3.24it/s, lr: 7.0e-05 loss: 1.719e-01] jam: 3%|▎ | 644/20000 [03:10<1:39:33, 3.24it/s, lr: 7.0e-05 loss: 2.209e-01] jam: 3%|▎ | 645/20000 [03:10<1:10:21, 4.59it/s, lr: 7.0e-05 loss: 2.209e-01] jam: 3%|▎ | 645/20000 [03:10<1:10:21, 4.59it/s, lr: 7.0e-05 loss: 4.160e-02] jam: 3%|▎ | 646/20000 [03:10<1:38:35, 3.27it/s, lr: 7.0e-05 loss: 4.160e-02] jam: 3%|▎ | 646/20000 [03:11<1:38:35, 3.27it/s, lr: 7.0e-05 loss: 2.013e-01] jam: 3%|▎ | 647/20000 [03:11<1:50:56, 2.91it/s, lr: 7.0e-05 loss: 2.013e-01] jam: 3%|▎ | 647/20000 [03:11<1:50:56, 2.91it/s, lr: 7.0e-05 loss: 8.101e-02] jam: 3%|▎ | 648/20000 [03:11<1:16:22, 4.22it/s, lr: 7.0e-05 loss: 8.101e-02] jam: 3%|▎ | 648/20000 [03:11<1:16:22, 4.22it/s, lr: 7.0e-05 loss: 1.445e-01] jam: 3%|▎ | 649/20000 [03:11<1:40:43, 3.20it/s, lr: 7.0e-05 loss: 1.445e-01] jam: 3%|▎ | 649/20000 [03:12<1:40:43, 3.20it/s, lr: 7.0e-05 loss: 1.129e-01] jam: 3%|▎ | 650/20000 [03:12<1:52:41, 2.86it/s, lr: 7.0e-05 loss: 1.129e-01] jam: 3%|▎ | 650/20000 [03:12<1:52:41, 2.86it/s, lr: 7.0e-05 loss: 2.266e-01] | |
| Saving at step 650 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000650.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 651/20000 [03:12<1:52:41, 2.86it/s, lr: 7.0e-05 loss: 2.896e-01] jam: 3%|▎ | 652/20000 [03:12<1:50:49, 2.91it/s, lr: 7.0e-05 loss: 2.896e-01] jam: 3%|▎ | 652/20000 [03:13<1:50:49, 2.91it/s, lr: 7.0e-05 loss: 1.592e-01] jam: 3%|▎ | 653/20000 [03:13<1:27:04, 3.70it/s, lr: 7.0e-05 loss: 1.592e-01] jam: 3%|▎ | 653/20000 [03:13<1:27:04, 3.70it/s, lr: 7.0e-05 loss: 1.924e-01] jam: 3%|▎ | 654/20000 [03:13<1:44:43, 3.08it/s, lr: 7.0e-05 loss: 1.924e-01] jam: 3%|▎ | 654/20000 [03:13<1:44:43, 3.08it/s, lr: 7.0e-05 loss: 9.639e-02] jam: 3%|▎ | 655/20000 [03:13<1:20:52, 3.99it/s, lr: 7.0e-05 loss: 9.639e-02] jam: 3%|▎ | 655/20000 [03:14<1:20:52, 3.99it/s, lr: 7.0e-05 loss: 6.197e-02] jam: 3%|▎ | 656/20000 [03:14<1:45:01, 3.07it/s, lr: 7.0e-05 loss: 6.197e-02] jam: 3%|▎ | 656/20000 [03:14<1:45:01, 3.07it/s, lr: 7.0e-05 loss: 1.284e-01] jam: 3%|▎ | 657/20000 [03:14<1:12:27, 4.45it/s, lr: 7.0e-05 loss: 1.284e-01] jam: 3%|▎ | 657/20000 [03:14<1:12:27, 4.45it/s, lr: 7.0e-05 loss: 2.305e-01] jam: 3%|▎ | 658/20000 [03:14<1:05:45, 4.90it/s, lr: 7.0e-05 loss: 2.305e-01] jam: 3%|▎ | 658/20000 [03:14<1:05:45, 4.90it/s, lr: 7.0e-05 loss: 1.121e-01] jam: 3%|▎ | 659/20000 [03:14<1:00:52, 5.30it/s, lr: 7.0e-05 loss: 1.121e-01] jam: 3%|▎ | 659/20000 [03:14<1:00:52, 5.30it/s, lr: 7.0e-05 loss: 2.646e-01] jam: 3%|▎ | 660/20000 [03:14<1:08:20, 4.72it/s, lr: 7.0e-05 loss: 2.646e-01] jam: 3%|▎ | 660/20000 [03:15<1:08:20, 4.72it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 3%|▎ | 661/20000 [03:15<1:39:21, 3.24it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 3%|▎ | 661/20000 [03:15<1:39:21, 3.24it/s, lr: 7.0e-05 loss: 1.769e-01] jam: 3%|▎ | 662/20000 [03:15<1:24:32, 3.81it/s, lr: 7.0e-05 loss: 1.769e-01] jam: 3%|▎ | 662/20000 [03:15<1:24:32, 3.81it/s, lr: 7.0e-05 loss: 1.187e-01] jam: 3%|▎ | 663/20000 [03:15<1:18:20, 4.11it/s, lr: 7.0e-05 loss: 1.187e-01] jam: 3%|▎ | 663/20000 [03:15<1:18:20, 4.11it/s, lr: 7.0e-05 loss: 1.036e-01] jam: 3%|▎ | 664/20000 [03:15<1:00:56, 5.29it/s, lr: 7.0e-05 loss: 1.036e-01] jam: 3%|▎ | 664/20000 [03:16<1:00:56, 5.29it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 3%|▎ | 665/20000 [03:16<1:01:57, 5.20it/s, lr: 7.0e-05 loss: 1.253e-01] jam: 3%|▎ | 665/20000 [03:16<1:01:57, 5.20it/s, lr: 7.0e-05 loss: 2.159e-01] jam: 3%|▎ | 666/20000 [03:16<1:33:42, 3.44it/s, lr: 7.0e-05 loss: 2.159e-01] jam: 3%|▎ | 666/20000 [03:16<1:33:42, 3.44it/s, lr: 7.0e-05 loss: 7.841e-02] jam: 3%|▎ | 667/20000 [03:16<1:44:56, 3.07it/s, lr: 7.0e-05 loss: 7.841e-02] jam: 3%|▎ | 667/20000 [03:17<1:44:56, 3.07it/s, lr: 7.0e-05 loss: 2.209e-01] jam: 3%|▎ | 668/20000 [03:17<1:52:09, 2.87it/s, lr: 7.0e-05 loss: 2.209e-01] jam: 3%|▎ | 668/20000 [03:17<1:52:09, 2.87it/s, lr: 7.0e-05 loss: 8.372e-02] jam: 3%|▎ | 669/20000 [03:17<1:50:18, 2.92it/s, lr: 7.0e-05 loss: 8.372e-02] jam: 3%|▎ | 669/20000 [03:17<1:50:18, 2.92it/s, lr: 7.0e-05 loss: 9.653e-02] jam: 3%|▎ | 670/20000 [03:17<1:13:52, 4.36it/s, lr: 7.0e-05 loss: 9.653e-02] jam: 3%|▎ | 670/20000 [03:17<1:13:52, 4.36it/s, lr: 7.0e-05 loss: 6.402e-02] jam: 3%|▎ | 671/20000 [03:17<1:00:25, 5.33it/s, lr: 7.0e-05 loss: 6.402e-02] jam: 3%|▎ | 671/20000 [03:18<1:00:25, 5.33it/s, lr: 7.0e-05 loss: 1.263e-01] jam: 3%|▎ | 672/20000 [03:18<1:34:29, 3.41it/s, lr: 7.0e-05 loss: 1.263e-01] jam: 3%|▎ | 672/20000 [03:18<1:34:29, 3.41it/s, lr: 7.0e-05 loss: 1.935e-01] jam: 3%|▎ | 673/20000 [03:18<1:05:23, 4.93it/s, lr: 7.0e-05 loss: 1.935e-01] jam: 3%|▎ | 673/20000 [03:18<1:05:23, 4.93it/s, lr: 7.0e-05 loss: 9.860e-02] jam: 3%|▎ | 674/20000 [03:18<1:10:09, 4.59it/s, lr: 7.0e-05 loss: 9.860e-02] jam: 3%|▎ | 674/20000 [03:18<1:10:09, 4.59it/s, lr: 7.0e-05 loss: 1.033e-01] jam: 3%|▎ | 675/20000 [03:18<1:37:55, 3.29it/s, lr: 7.0e-05 loss: 1.033e-01] jam: 3%|▎ | 675/20000 [03:19<1:37:55, 3.29it/s, lr: 7.0e-05 loss: 9.960e-02] | |
| Saving at step 675 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000675.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 3%|▎ | 676/20000 [03:19<1:37:55, 3.29it/s, lr: 7.0e-05 loss: 1.741e-01] jam: 3%|▎ | 677/20000 [03:19<1:16:35, 4.20it/s, lr: 7.0e-05 loss: 1.741e-01] jam: 3%|▎ | 677/20000 [03:19<1:16:35, 4.20it/s, lr: 7.0e-05 loss: 8.603e-02] jam: 3%|▎ | 678/20000 [03:19<1:35:50, 3.36it/s, lr: 7.0e-05 loss: 8.603e-02] jam: 3%|▎ | 678/20000 [03:20<1:35:50, 3.36it/s, lr: 7.0e-05 loss: 1.242e-01] jam: 3%|▎ | 679/20000 [03:20<1:20:09, 4.02it/s, lr: 7.0e-05 loss: 1.242e-01] jam: 3%|▎ | 679/20000 [03:20<1:20:09, 4.02it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 3%|▎ | 680/20000 [03:20<1:44:24, 3.08it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 3%|▎ | 680/20000 [03:20<1:44:24, 3.08it/s, lr: 7.0e-05 loss: 2.636e-01] jam: 3%|▎ | 681/20000 [03:20<1:50:33, 2.91it/s, lr: 7.0e-05 loss: 2.636e-01] jam: 3%|▎ | 681/20000 [03:21<1:50:33, 2.91it/s, lr: 7.0e-05 loss: 4.700e-02] jam: 3%|▎ | 682/20000 [03:21<1:53:20, 2.84it/s, lr: 7.0e-05 loss: 4.700e-02] jam: 3%|▎ | 682/20000 [03:21<1:53:20, 2.84it/s, lr: 7.0e-05 loss: 1.078e-01] jam: 3%|▎ | 683/20000 [03:21<1:25:10, 3.78it/s, lr: 7.0e-05 loss: 1.078e-01] jam: 3%|▎ | 683/20000 [03:21<1:25:10, 3.78it/s, lr: 7.0e-05 loss: 7.175e-02] jam: 3%|▎ | 684/20000 [03:21<1:46:42, 3.02it/s, lr: 7.0e-05 loss: 7.175e-02] jam: 3%|▎ | 684/20000 [03:22<1:46:42, 3.02it/s, lr: 7.0e-05 loss: 1.817e-01] jam: 3%|▎ | 685/20000 [03:22<1:13:40, 4.37it/s, lr: 7.0e-05 loss: 1.817e-01] jam: 3%|▎ | 685/20000 [03:22<1:13:40, 4.37it/s, lr: 7.0e-05 loss: 5.661e-02] jam: 3%|▎ | 686/20000 [03:22<1:39:40, 3.23it/s, lr: 7.0e-05 loss: 5.661e-02] jam: 3%|▎ | 686/20000 [03:22<1:39:40, 3.23it/s, lr: 7.0e-05 loss: 1.916e-01] jam: 3%|▎ | 687/20000 [03:22<1:18:57, 4.08it/s, lr: 7.0e-05 loss: 1.916e-01] jam: 3%|▎ | 687/20000 [03:22<1:18:57, 4.08it/s, lr: 7.0e-05 loss: 9.730e-02] jam: 3%|▎ | 688/20000 [03:22<1:10:41, 4.55it/s, lr: 7.0e-05 loss: 9.730e-02] jam: 3%|▎ | 688/20000 [03:22<1:10:41, 4.55it/s, lr: 7.0e-05 loss: 1.295e-01] jam: 3%|▎ | 689/20000 [03:22<1:00:10, 5.35it/s, lr: 7.0e-05 loss: 1.295e-01] jam: 3%|▎ | 689/20000 [03:23<1:00:10, 5.35it/s, lr: 7.0e-05 loss: 1.236e-01] jam: 3%|▎ | 690/20000 [03:23<1:14:42, 4.31it/s, lr: 7.0e-05 loss: 1.236e-01] jam: 3%|▎ | 690/20000 [03:23<1:14:42, 4.31it/s, lr: 7.0e-05 loss: 8.647e-02] jam: 3%|▎ | 691/20000 [03:23<1:39:19, 3.24it/s, lr: 7.0e-05 loss: 8.647e-02] jam: 3%|▎ | 691/20000 [03:23<1:39:19, 3.24it/s, lr: 7.0e-05 loss: 7.891e-02] jam: 3%|▎ | 692/20000 [03:23<1:52:17, 2.87it/s, lr: 7.0e-05 loss: 7.891e-02] jam: 3%|▎ | 692/20000 [03:24<1:52:17, 2.87it/s, lr: 7.0e-05 loss: 1.755e-01] jam: 3%|▎ | 693/20000 [03:24<1:14:30, 4.32it/s, lr: 7.0e-05 loss: 1.755e-01] jam: 3%|▎ | 693/20000 [03:24<1:14:30, 4.32it/s, lr: 7.0e-05 loss: 7.272e-02] jam: 3%|▎ | 694/20000 [03:24<1:39:45, 3.23it/s, lr: 7.0e-05 loss: 7.272e-02] jam: 3%|▎ | 694/20000 [03:24<1:39:45, 3.23it/s, lr: 7.0e-05 loss: 1.085e-01] jam: 3%|▎ | 695/20000 [03:24<1:34:41, 3.40it/s, lr: 7.0e-05 loss: 1.085e-01] jam: 3%|▎ | 695/20000 [03:25<1:34:41, 3.40it/s, lr: 7.0e-05 loss: 2.066e-01] jam: 3%|▎ | 696/20000 [03:25<1:45:01, 3.06it/s, lr: 7.0e-05 loss: 2.066e-01] jam: 3%|▎ | 696/20000 [03:25<1:45:01, 3.06it/s, lr: 7.0e-05 loss: 2.701e-01] jam: 3%|▎ | 697/20000 [03:25<1:17:16, 4.16it/s, lr: 7.0e-05 loss: 2.701e-01] jam: 3%|▎ | 697/20000 [03:25<1:17:16, 4.16it/s, lr: 7.0e-05 loss: 1.398e-01] jam: 3%|▎ | 698/20000 [03:25<1:42:38, 3.13it/s, lr: 7.0e-05 loss: 1.398e-01] jam: 3%|▎ | 698/20000 [03:25<1:42:38, 3.13it/s, lr: 7.0e-05 loss: 1.778e-01] jam: 3%|▎ | 699/20000 [03:25<1:47:40, 2.99it/s, lr: 7.0e-05 loss: 1.778e-01] jam: 3%|▎ | 699/20000 [03:26<1:47:40, 2.99it/s, lr: 7.0e-05 loss: 6.299e-02] jam: 4%|▎ | 700/20000 [03:26<1:51:03, 2.90it/s, lr: 7.0e-05 loss: 6.299e-02] jam: 4%|▎ | 700/20000 [03:26<1:51:03, 2.90it/s, lr: 7.0e-05 loss: 8.735e-02] | |
| Saving at step 700 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000700.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▎ | 701/20000 [03:27<1:51:03, 2.90it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 4%|▎ | 702/20000 [03:27<1:21:41, 3.94it/s, lr: 7.0e-05 loss: 1.079e-01] jam: 4%|▎ | 702/20000 [03:27<1:21:41, 3.94it/s, lr: 7.0e-05 loss: 1.500e-01] jam: 4%|▎ | 703/20000 [03:27<1:16:13, 4.22it/s, lr: 7.0e-05 loss: 1.500e-01] jam: 4%|▎ | 703/20000 [03:27<1:16:13, 4.22it/s, lr: 7.0e-05 loss: 2.516e-01] jam: 4%|▎ | 704/20000 [03:27<1:24:58, 3.78it/s, lr: 7.0e-05 loss: 2.516e-01] jam: 4%|▎ | 704/20000 [03:27<1:24:58, 3.78it/s, lr: 7.0e-05 loss: 5.233e-02] jam: 4%|▎ | 705/20000 [03:27<1:50:09, 2.92it/s, lr: 7.0e-05 loss: 5.233e-02] jam: 4%|▎ | 705/20000 [03:28<1:50:09, 2.92it/s, lr: 7.0e-05 loss: 6.737e-02] jam: 4%|▎ | 706/20000 [03:28<1:55:42, 2.78it/s, lr: 7.0e-05 loss: 6.737e-02] jam: 4%|▎ | 706/20000 [03:28<1:55:42, 2.78it/s, lr: 7.0e-05 loss: 1.334e-01] jam: 4%|▎ | 707/20000 [03:28<1:15:16, 4.27it/s, lr: 7.0e-05 loss: 1.334e-01] jam: 4%|▎ | 707/20000 [03:28<1:15:16, 4.27it/s, lr: 7.0e-05 loss: 1.066e-01] jam: 4%|▎ | 708/20000 [03:28<1:38:16, 3.27it/s, lr: 7.0e-05 loss: 1.066e-01] jam: 4%|▎ | 708/20000 [03:28<1:38:16, 3.27it/s, lr: 7.0e-05 loss: 8.270e-02] jam: 4%|▎ | 709/20000 [03:28<1:10:08, 4.58it/s, lr: 7.0e-05 loss: 8.270e-02] jam: 4%|▎ | 709/20000 [03:29<1:10:08, 4.58it/s, lr: 7.0e-05 loss: 1.296e-01] jam: 4%|▎ | 710/20000 [03:29<1:12:17, 4.45it/s, lr: 7.0e-05 loss: 1.296e-01] jam: 4%|▎ | 710/20000 [03:29<1:12:17, 4.45it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 4%|▎ | 711/20000 [03:29<1:23:44, 3.84it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 4%|▎ | 711/20000 [03:29<1:23:44, 3.84it/s, lr: 7.0e-05 loss: 2.893e-01] jam: 4%|▎ | 712/20000 [03:29<1:06:21, 4.84it/s, lr: 7.0e-05 loss: 2.893e-01] jam: 4%|▎ | 712/20000 [03:29<1:06:21, 4.84it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 4%|▎ | 713/20000 [03:29<1:03:54, 5.03it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 4%|▎ | 713/20000 [03:30<1:03:54, 5.03it/s, lr: 7.0e-05 loss: 1.377e-01] jam: 4%|▎ | 714/20000 [03:30<1:05:47, 4.89it/s, lr: 7.0e-05 loss: 1.377e-01] jam: 4%|▎ | 714/20000 [03:30<1:05:47, 4.89it/s, lr: 7.0e-05 loss: 5.929e-02] jam: 4%|▎ | 715/20000 [03:30<1:03:07, 5.09it/s, lr: 7.0e-05 loss: 5.929e-02] jam: 4%|▎ | 715/20000 [03:30<1:03:07, 5.09it/s, lr: 7.0e-05 loss: 3.102e-01] jam: 4%|▎ | 716/20000 [03:30<1:01:48, 5.20it/s, lr: 7.0e-05 loss: 3.102e-01] jam: 4%|▎ | 716/20000 [03:30<1:01:48, 5.20it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 4%|▎ | 717/20000 [03:30<1:09:45, 4.61it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 4%|▎ | 717/20000 [03:30<1:09:45, 4.61it/s, lr: 7.0e-05 loss: 1.153e-01] jam: 4%|▎ | 718/20000 [03:30<1:11:11, 4.51it/s, lr: 7.0e-05 loss: 1.153e-01] jam: 4%|▎ | 718/20000 [03:31<1:11:11, 4.51it/s, lr: 7.0e-05 loss: 1.421e-01] jam: 4%|▎ | 719/20000 [03:31<1:04:50, 4.96it/s, lr: 7.0e-05 loss: 1.421e-01] jam: 4%|▎ | 719/20000 [03:31<1:04:50, 4.96it/s, lr: 7.0e-05 loss: 2.098e-01] jam: 4%|▎ | 720/20000 [03:31<1:35:39, 3.36it/s, lr: 7.0e-05 loss: 2.098e-01] jam: 4%|▎ | 720/20000 [03:31<1:35:39, 3.36it/s, lr: 7.0e-05 loss: 1.817e-01] jam: 4%|▎ | 721/20000 [03:31<1:18:20, 4.10it/s, lr: 7.0e-05 loss: 1.817e-01] jam: 4%|▎ | 721/20000 [03:31<1:18:20, 4.10it/s, lr: 7.0e-05 loss: 1.798e-01] jam: 4%|▎ | 722/20000 [03:31<1:09:58, 4.59it/s, lr: 7.0e-05 loss: 1.798e-01] jam: 4%|▎ | 722/20000 [03:32<1:09:58, 4.59it/s, lr: 7.0e-05 loss: 1.454e-01] jam: 4%|▎ | 723/20000 [03:32<1:05:44, 4.89it/s, lr: 7.0e-05 loss: 1.454e-01] jam: 4%|▎ | 723/20000 [03:32<1:05:44, 4.89it/s, lr: 7.0e-05 loss: 1.258e-01] jam: 4%|▎ | 724/20000 [03:32<1:02:41, 5.12it/s, lr: 7.0e-05 loss: 1.258e-01] jam: 4%|▎ | 724/20000 [03:32<1:02:41, 5.12it/s, lr: 7.0e-05 loss: 5.589e-02] jam: 4%|▎ | 725/20000 [03:32<1:05:13, 4.93it/s, lr: 7.0e-05 loss: 5.589e-02] jam: 4%|▎ | 725/20000 [03:32<1:05:13, 4.93it/s, lr: 7.0e-05 loss: 1.066e-01] | |
| Saving at step 725 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000725.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▎ | 726/20000 [03:33<1:05:13, 4.93it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 4%|▎ | 727/20000 [03:33<1:35:24, 3.37it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 4%|▎ | 727/20000 [03:33<1:35:24, 3.37it/s, lr: 7.0e-05 loss: 2.647e-01] jam: 4%|▎ | 728/20000 [03:33<1:50:04, 2.92it/s, lr: 7.0e-05 loss: 2.647e-01] jam: 4%|▎ | 728/20000 [03:34<1:50:04, 2.92it/s, lr: 7.0e-05 loss: 4.825e-02] jam: 4%|▎ | 729/20000 [03:34<1:47:50, 2.98it/s, lr: 7.0e-05 loss: 4.825e-02] jam: 4%|▎ | 729/20000 [03:34<1:47:50, 2.98it/s, lr: 7.0e-05 loss: 1.735e-01] jam: 4%|▎ | 730/20000 [03:34<1:54:16, 2.81it/s, lr: 7.0e-05 loss: 1.735e-01] jam: 4%|▎ | 730/20000 [03:34<1:54:16, 2.81it/s, lr: 7.0e-05 loss: 5.266e-02] jam: 4%|▎ | 731/20000 [03:34<1:51:04, 2.89it/s, lr: 7.0e-05 loss: 5.266e-02] jam: 4%|▎ | 731/20000 [03:35<1:51:04, 2.89it/s, lr: 7.0e-05 loss: 2.306e-01] jam: 4%|▎ | 732/20000 [03:35<1:16:05, 4.22it/s, lr: 7.0e-05 loss: 2.306e-01] jam: 4%|▎ | 732/20000 [03:35<1:16:05, 4.22it/s, lr: 7.0e-05 loss: 2.660e-01] jam: 4%|▎ | 733/20000 [03:35<1:05:12, 4.92it/s, lr: 7.0e-05 loss: 2.660e-01] jam: 4%|▎ | 733/20000 [03:35<1:05:12, 4.92it/s, lr: 7.0e-05 loss: 1.974e-01] jam: 4%|▎ | 734/20000 [03:35<1:35:27, 3.36it/s, lr: 7.0e-05 loss: 1.974e-01] jam: 4%|▎ | 734/20000 [03:35<1:35:27, 3.36it/s, lr: 7.0e-05 loss: 1.954e-01] jam: 4%|▎ | 735/20000 [03:35<1:31:49, 3.50it/s, lr: 7.0e-05 loss: 1.954e-01] jam: 4%|▎ | 735/20000 [03:36<1:31:49, 3.50it/s, lr: 7.0e-05 loss: 7.125e-02] jam: 4%|▎ | 736/20000 [03:36<1:48:44, 2.95it/s, lr: 7.0e-05 loss: 7.125e-02] jam: 4%|▎ | 736/20000 [03:36<1:48:44, 2.95it/s, lr: 7.0e-05 loss: 2.812e-01] jam: 4%|▎ | 737/20000 [03:36<1:49:17, 2.94it/s, lr: 7.0e-05 loss: 2.812e-01] jam: 4%|▎ | 737/20000 [03:36<1:49:17, 2.94it/s, lr: 7.0e-05 loss: 1.986e-01] jam: 4%|▎ | 738/20000 [03:36<1:21:30, 3.94it/s, lr: 7.0e-05 loss: 1.986e-01] jam: 4%|▎ | 738/20000 [03:37<1:21:30, 3.94it/s, lr: 7.0e-05 loss: 2.427e-01] jam: 4%|▎ | 739/20000 [03:37<1:41:58, 3.15it/s, lr: 7.0e-05 loss: 2.427e-01] jam: 4%|▎ | 739/20000 [03:37<1:41:58, 3.15it/s, lr: 7.0e-05 loss: 1.429e-01] jam: 4%|▎ | 740/20000 [03:37<1:12:41, 4.42it/s, lr: 7.0e-05 loss: 1.429e-01] jam: 4%|▎ | 740/20000 [03:37<1:12:41, 4.42it/s, lr: 7.0e-05 loss: 1.776e-01] jam: 4%|▎ | 741/20000 [03:37<1:08:26, 4.69it/s, lr: 7.0e-05 loss: 1.776e-01] jam: 4%|▎ | 741/20000 [03:37<1:08:26, 4.69it/s, lr: 7.0e-05 loss: 8.458e-02] jam: 4%|▎ | 742/20000 [03:37<1:05:04, 4.93it/s, lr: 7.0e-05 loss: 8.458e-02] jam: 4%|▎ | 742/20000 [03:37<1:05:04, 4.93it/s, lr: 7.0e-05 loss: 1.734e-01] jam: 4%|▎ | 743/20000 [03:37<1:14:11, 4.33it/s, lr: 7.0e-05 loss: 1.734e-01] jam: 4%|▎ | 743/20000 [03:38<1:14:11, 4.33it/s, lr: 7.0e-05 loss: 2.193e-01] jam: 4%|▎ | 744/20000 [03:38<1:43:59, 3.09it/s, lr: 7.0e-05 loss: 2.193e-01] jam: 4%|▎ | 744/20000 [03:38<1:43:59, 3.09it/s, lr: 7.0e-05 loss: 1.958e-01] jam: 4%|▎ | 745/20000 [03:38<1:12:29, 4.43it/s, lr: 7.0e-05 loss: 1.958e-01] jam: 4%|▎ | 745/20000 [03:38<1:12:29, 4.43it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▎ | 746/20000 [03:38<1:06:17, 4.84it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▎ | 746/20000 [03:39<1:06:17, 4.84it/s, lr: 7.0e-05 loss: 1.666e-01] jam: 4%|▎ | 747/20000 [03:39<1:35:05, 3.37it/s, lr: 7.0e-05 loss: 1.666e-01] jam: 4%|▎ | 747/20000 [03:39<1:35:05, 3.37it/s, lr: 7.0e-05 loss: 5.135e-02] jam: 4%|▎ | 748/20000 [03:39<1:07:26, 4.76it/s, lr: 7.0e-05 loss: 5.135e-02] jam: 4%|▎ | 748/20000 [03:39<1:07:26, 4.76it/s, lr: 7.0e-05 loss: 1.326e-01] jam: 4%|▎ | 749/20000 [03:39<1:35:44, 3.35it/s, lr: 7.0e-05 loss: 1.326e-01] jam: 4%|▎ | 749/20000 [03:39<1:35:44, 3.35it/s, lr: 7.0e-05 loss: 3.047e-01] jam: 4%|▍ | 750/20000 [03:39<1:11:18, 4.50it/s, lr: 7.0e-05 loss: 3.047e-01] jam: 4%|▍ | 750/20000 [03:40<1:11:18, 4.50it/s, lr: 7.0e-05 loss: 1.047e-01] | |
| Saving at step 750 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000750.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 751/20000 [03:40<1:11:18, 4.50it/s, lr: 7.0e-05 loss: 8.928e-02] jam: 4%|▍ | 752/20000 [03:40<1:11:04, 4.51it/s, lr: 7.0e-05 loss: 8.928e-02] jam: 4%|▍ | 752/20000 [03:40<1:11:04, 4.51it/s, lr: 7.0e-05 loss: 1.719e-01] jam: 4%|▍ | 753/20000 [03:40<1:06:01, 4.86it/s, lr: 7.0e-05 loss: 1.719e-01] jam: 4%|▍ | 753/20000 [03:41<1:06:01, 4.86it/s, lr: 7.0e-05 loss: 1.327e-01] jam: 4%|▍ | 754/20000 [03:41<1:40:30, 3.19it/s, lr: 7.0e-05 loss: 1.327e-01] jam: 4%|▍ | 754/20000 [03:41<1:40:30, 3.19it/s, lr: 7.0e-05 loss: 2.730e-02] jam: 4%|▍ | 755/20000 [03:41<1:10:41, 4.54it/s, lr: 7.0e-05 loss: 2.730e-02] jam: 4%|▍ | 755/20000 [03:41<1:10:41, 4.54it/s, lr: 7.0e-05 loss: 1.324e-01] jam: 4%|▍ | 756/20000 [03:41<1:03:53, 5.02it/s, lr: 7.0e-05 loss: 1.324e-01] jam: 4%|▍ | 756/20000 [03:41<1:03:53, 5.02it/s, lr: 7.0e-05 loss: 2.924e-01] jam: 4%|▍ | 757/20000 [03:41<1:05:05, 4.93it/s, lr: 7.0e-05 loss: 2.924e-01] jam: 4%|▍ | 757/20000 [03:41<1:05:05, 4.93it/s, lr: 7.0e-05 loss: 1.432e-01] jam: 4%|▍ | 758/20000 [03:41<1:36:09, 3.33it/s, lr: 7.0e-05 loss: 1.432e-01] jam: 4%|▍ | 758/20000 [03:42<1:36:09, 3.33it/s, lr: 7.0e-05 loss: 7.317e-02] jam: 4%|▍ | 759/20000 [03:42<1:05:56, 4.86it/s, lr: 7.0e-05 loss: 7.317e-02] jam: 4%|▍ | 759/20000 [03:42<1:05:56, 4.86it/s, lr: 7.0e-05 loss: 5.271e-02] jam: 4%|▍ | 760/20000 [03:42<59:55, 5.35it/s, lr: 7.0e-05 loss: 5.271e-02] jam: 4%|▍ | 760/20000 [03:42<59:55, 5.35it/s, lr: 7.0e-05 loss: 1.001e-01] jam: 4%|▍ | 761/20000 [03:42<1:38:39, 3.25it/s, lr: 7.0e-05 loss: 1.001e-01] jam: 4%|▍ | 761/20000 [03:42<1:38:39, 3.25it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 4%|▍ | 762/20000 [03:42<1:18:48, 4.07it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 4%|▍ | 762/20000 [03:43<1:18:48, 4.07it/s, lr: 7.0e-05 loss: 1.443e-01] jam: 4%|▍ | 763/20000 [03:43<1:42:38, 3.12it/s, lr: 7.0e-05 loss: 1.443e-01] jam: 4%|▍ | 763/20000 [03:43<1:42:38, 3.12it/s, lr: 7.0e-05 loss: 1.330e-01] jam: 4%|▍ | 764/20000 [03:43<1:46:24, 3.01it/s, lr: 7.0e-05 loss: 1.330e-01] jam: 4%|▍ | 764/20000 [03:43<1:46:24, 3.01it/s, lr: 7.0e-05 loss: 8.487e-02] jam: 4%|▍ | 765/20000 [03:43<1:53:18, 2.83it/s, lr: 7.0e-05 loss: 8.487e-02] jam: 4%|▍ | 765/20000 [03:44<1:53:18, 2.83it/s, lr: 7.0e-05 loss: 1.269e-01] jam: 4%|▍ | 766/20000 [03:44<1:49:56, 2.92it/s, lr: 7.0e-05 loss: 1.269e-01] jam: 4%|▍ | 766/20000 [03:44<1:49:56, 2.92it/s, lr: 7.0e-05 loss: 1.044e-01] jam: 4%|▍ | 767/20000 [03:44<1:21:59, 3.91it/s, lr: 7.0e-05 loss: 1.044e-01] jam: 4%|▍ | 767/20000 [03:44<1:21:59, 3.91it/s, lr: 7.0e-05 loss: 2.394e-01] jam: 4%|▍ | 768/20000 [03:44<1:09:15, 4.63it/s, lr: 7.0e-05 loss: 2.394e-01] jam: 4%|▍ | 768/20000 [03:45<1:09:15, 4.63it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 4%|▍ | 769/20000 [03:45<1:42:10, 3.14it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 4%|▍ | 769/20000 [03:45<1:42:10, 3.14it/s, lr: 7.0e-05 loss: 1.289e-01] jam: 4%|▍ | 770/20000 [03:45<1:46:54, 3.00it/s, lr: 7.0e-05 loss: 1.289e-01] jam: 4%|▍ | 770/20000 [03:45<1:46:54, 3.00it/s, lr: 7.0e-05 loss: 1.603e-01] jam: 4%|▍ | 771/20000 [03:45<1:18:07, 4.10it/s, lr: 7.0e-05 loss: 1.603e-01] jam: 4%|▍ | 771/20000 [03:45<1:18:07, 4.10it/s, lr: 7.0e-05 loss: 2.023e-01] jam: 4%|▍ | 772/20000 [03:45<1:11:00, 4.51it/s, lr: 7.0e-05 loss: 2.023e-01] jam: 4%|▍ | 772/20000 [03:46<1:11:00, 4.51it/s, lr: 7.0e-05 loss: 2.307e-01] jam: 4%|▍ | 773/20000 [03:46<1:08:20, 4.69it/s, lr: 7.0e-05 loss: 2.307e-01] jam: 4%|▍ | 773/20000 [03:46<1:08:20, 4.69it/s, lr: 7.0e-05 loss: 1.721e-01] jam: 4%|▍ | 774/20000 [03:46<1:40:44, 3.18it/s, lr: 7.0e-05 loss: 1.721e-01] jam: 4%|▍ | 774/20000 [03:46<1:40:44, 3.18it/s, lr: 7.0e-05 loss: 7.027e-02] jam: 4%|▍ | 775/20000 [03:46<1:19:34, 4.03it/s, lr: 7.0e-05 loss: 7.027e-02] jam: 4%|▍ | 775/20000 [03:46<1:19:34, 4.03it/s, lr: 7.0e-05 loss: 1.347e-01] | |
| Saving at step 775 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000775.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 776/20000 [03:47<1:19:34, 4.03it/s, lr: 7.0e-05 loss: 3.836e-01] jam: 4%|▍ | 777/20000 [03:47<1:12:22, 4.43it/s, lr: 7.0e-05 loss: 3.836e-01] jam: 4%|▍ | 777/20000 [03:47<1:12:22, 4.43it/s, lr: 7.0e-05 loss: 6.782e-02] jam: 4%|▍ | 778/20000 [03:47<1:10:41, 4.53it/s, lr: 7.0e-05 loss: 6.782e-02] jam: 4%|▍ | 778/20000 [03:47<1:10:41, 4.53it/s, lr: 7.0e-05 loss: 1.604e-01] jam: 4%|▍ | 779/20000 [03:47<1:02:05, 5.16it/s, lr: 7.0e-05 loss: 1.604e-01] jam: 4%|▍ | 779/20000 [03:48<1:02:05, 5.16it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 4%|▍ | 780/20000 [03:48<1:33:53, 3.41it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 4%|▍ | 780/20000 [03:48<1:33:53, 3.41it/s, lr: 7.0e-05 loss: 1.549e-01] jam: 4%|▍ | 781/20000 [03:48<1:47:49, 2.97it/s, lr: 7.0e-05 loss: 1.549e-01] jam: 4%|▍ | 781/20000 [03:48<1:47:49, 2.97it/s, lr: 7.0e-05 loss: 9.875e-02] jam: 4%|▍ | 782/20000 [03:48<1:48:52, 2.94it/s, lr: 7.0e-05 loss: 9.875e-02] jam: 4%|▍ | 782/20000 [03:49<1:48:52, 2.94it/s, lr: 7.0e-05 loss: 1.304e-01] jam: 4%|▍ | 783/20000 [03:49<1:19:26, 4.03it/s, lr: 7.0e-05 loss: 1.304e-01] jam: 4%|▍ | 783/20000 [03:49<1:19:26, 4.03it/s, lr: 7.0e-05 loss: 1.815e-01] jam: 4%|▍ | 784/20000 [03:49<1:06:03, 4.85it/s, lr: 7.0e-05 loss: 1.815e-01] jam: 4%|▍ | 784/20000 [03:49<1:06:03, 4.85it/s, lr: 7.0e-05 loss: 9.074e-02] jam: 4%|▍ | 785/20000 [03:49<1:01:36, 5.20it/s, lr: 7.0e-05 loss: 9.074e-02] jam: 4%|▍ | 785/20000 [03:49<1:01:36, 5.20it/s, lr: 7.0e-05 loss: 1.633e-01] jam: 4%|▍ | 786/20000 [03:49<57:40, 5.55it/s, lr: 7.0e-05 loss: 1.633e-01] jam: 4%|▍ | 786/20000 [03:49<57:40, 5.55it/s, lr: 7.0e-05 loss: 1.692e-01] jam: 4%|▍ | 787/20000 [03:49<55:25, 5.78it/s, lr: 7.0e-05 loss: 1.692e-01] jam: 4%|▍ | 787/20000 [03:50<55:25, 5.78it/s, lr: 7.0e-05 loss: 1.546e-01] jam: 4%|▍ | 788/20000 [03:50<1:33:43, 3.42it/s, lr: 7.0e-05 loss: 1.546e-01] jam: 4%|▍ | 788/20000 [03:50<1:33:43, 3.42it/s, lr: 7.0e-05 loss: 2.157e-01] jam: 4%|▍ | 789/20000 [03:50<1:16:12, 4.20it/s, lr: 7.0e-05 loss: 2.157e-01] jam: 4%|▍ | 789/20000 [03:50<1:16:12, 4.20it/s, lr: 7.0e-05 loss: 6.111e-02] jam: 4%|▍ | 790/20000 [03:50<1:08:35, 4.67it/s, lr: 7.0e-05 loss: 6.111e-02] jam: 4%|▍ | 790/20000 [03:50<1:08:35, 4.67it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 4%|▍ | 791/20000 [03:50<1:03:21, 5.05it/s, lr: 7.0e-05 loss: 1.706e-01] jam: 4%|▍ | 791/20000 [03:50<1:03:21, 5.05it/s, lr: 7.0e-05 loss: 3.175e-02] jam: 4%|▍ | 792/20000 [03:50<1:06:41, 4.80it/s, lr: 7.0e-05 loss: 3.175e-02] jam: 4%|▍ | 792/20000 [03:51<1:06:41, 4.80it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 4%|▍ | 793/20000 [03:51<1:13:28, 4.36it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 4%|▍ | 793/20000 [03:51<1:13:28, 4.36it/s, lr: 7.0e-05 loss: 1.894e-01] jam: 4%|▍ | 794/20000 [03:51<1:40:09, 3.20it/s, lr: 7.0e-05 loss: 1.894e-01] jam: 4%|▍ | 794/20000 [03:51<1:40:09, 3.20it/s, lr: 7.0e-05 loss: 1.283e-01] jam: 4%|▍ | 795/20000 [03:51<1:10:42, 4.53it/s, lr: 7.0e-05 loss: 1.283e-01] jam: 4%|▍ | 795/20000 [03:52<1:10:42, 4.53it/s, lr: 7.0e-05 loss: 9.750e-02] jam: 4%|▍ | 796/20000 [03:52<1:37:23, 3.29it/s, lr: 7.0e-05 loss: 9.750e-02] jam: 4%|▍ | 796/20000 [03:52<1:37:23, 3.29it/s, lr: 7.0e-05 loss: 1.471e-01] jam: 4%|▍ | 797/20000 [03:52<1:49:07, 2.93it/s, lr: 7.0e-05 loss: 1.471e-01] jam: 4%|▍ | 797/20000 [03:52<1:49:07, 2.93it/s, lr: 7.0e-05 loss: 1.534e-01] jam: 4%|▍ | 798/20000 [03:52<1:47:30, 2.98it/s, lr: 7.0e-05 loss: 1.534e-01] jam: 4%|▍ | 798/20000 [03:53<1:47:30, 2.98it/s, lr: 7.0e-05 loss: 7.580e-02] jam: 4%|▍ | 799/20000 [03:53<1:53:22, 2.82it/s, lr: 7.0e-05 loss: 7.580e-02] jam: 4%|▍ | 799/20000 [03:53<1:53:22, 2.82it/s, lr: 7.0e-05 loss: 1.539e-01] jam: 4%|▍ | 800/20000 [03:53<1:13:18, 4.36it/s, lr: 7.0e-05 loss: 1.539e-01] jam: 4%|▍ | 800/20000 [03:53<1:13:18, 4.36it/s, lr: 7.0e-05 loss: 7.586e-02] | |
| Saving at step 800 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000800.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 801/20000 [03:54<1:13:18, 4.36it/s, lr: 7.0e-05 loss: 2.102e-01] jam: 4%|▍ | 802/20000 [03:54<1:12:22, 4.42it/s, lr: 7.0e-05 loss: 2.102e-01] jam: 4%|▍ | 802/20000 [03:54<1:12:22, 4.42it/s, lr: 7.0e-05 loss: 9.399e-02] jam: 4%|▍ | 803/20000 [03:54<1:02:12, 5.14it/s, lr: 7.0e-05 loss: 9.399e-02] jam: 4%|▍ | 803/20000 [03:54<1:02:12, 5.14it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 4%|▍ | 804/20000 [03:54<1:32:16, 3.47it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 4%|▍ | 804/20000 [03:54<1:32:16, 3.47it/s, lr: 7.0e-05 loss: 8.460e-02] jam: 4%|▍ | 805/20000 [03:54<1:43:24, 3.09it/s, lr: 7.0e-05 loss: 8.460e-02] jam: 4%|▍ | 805/20000 [03:55<1:43:24, 3.09it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 4%|▍ | 806/20000 [03:55<1:13:09, 4.37it/s, lr: 7.0e-05 loss: 1.368e-01] jam: 4%|▍ | 806/20000 [03:55<1:13:09, 4.37it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 4%|▍ | 807/20000 [03:55<1:38:11, 3.26it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 4%|▍ | 807/20000 [03:55<1:38:11, 3.26it/s, lr: 7.0e-05 loss: 1.251e-01] jam: 4%|▍ | 808/20000 [03:55<1:15:57, 4.21it/s, lr: 7.0e-05 loss: 1.251e-01] jam: 4%|▍ | 808/20000 [03:55<1:15:57, 4.21it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 4%|▍ | 809/20000 [03:55<1:10:55, 4.51it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 4%|▍ | 809/20000 [03:56<1:10:55, 4.51it/s, lr: 7.0e-05 loss: 2.014e-01] jam: 4%|▍ | 810/20000 [03:56<1:40:08, 3.19it/s, lr: 7.0e-05 loss: 2.014e-01] jam: 4%|▍ | 810/20000 [03:56<1:40:08, 3.19it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 4%|▍ | 811/20000 [03:56<1:14:46, 4.28it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 4%|▍ | 811/20000 [03:56<1:14:46, 4.28it/s, lr: 7.0e-05 loss: 1.411e-01] jam: 4%|▍ | 812/20000 [03:56<1:14:14, 4.31it/s, lr: 7.0e-05 loss: 1.411e-01] jam: 4%|▍ | 812/20000 [03:56<1:14:14, 4.31it/s, lr: 7.0e-05 loss: 5.461e-02] jam: 4%|▍ | 813/20000 [03:56<1:10:12, 4.56it/s, lr: 7.0e-05 loss: 5.461e-02] jam: 4%|▍ | 813/20000 [03:57<1:10:12, 4.56it/s, lr: 7.0e-05 loss: 1.786e-01] jam: 4%|▍ | 814/20000 [03:57<1:41:00, 3.17it/s, lr: 7.0e-05 loss: 1.786e-01] jam: 4%|▍ | 814/20000 [03:57<1:41:00, 3.17it/s, lr: 7.0e-05 loss: 6.111e-02] jam: 4%|▍ | 815/20000 [03:57<1:46:02, 3.02it/s, lr: 7.0e-05 loss: 6.111e-02] jam: 4%|▍ | 815/20000 [03:57<1:46:02, 3.02it/s, lr: 7.0e-05 loss: 1.664e-01] jam: 4%|▍ | 816/20000 [03:57<1:18:22, 4.08it/s, lr: 7.0e-05 loss: 1.664e-01] jam: 4%|▍ | 816/20000 [03:57<1:18:22, 4.08it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 4%|▍ | 817/20000 [03:57<1:13:33, 4.35it/s, lr: 7.0e-05 loss: 1.465e-01] jam: 4%|▍ | 817/20000 [03:58<1:13:33, 4.35it/s, lr: 7.0e-05 loss: 9.029e-02] jam: 4%|▍ | 818/20000 [03:58<1:42:40, 3.11it/s, lr: 7.0e-05 loss: 9.029e-02] jam: 4%|▍ | 818/20000 [03:58<1:42:40, 3.11it/s, lr: 7.0e-05 loss: 1.267e-01] jam: 4%|▍ | 819/20000 [03:58<1:20:21, 3.98it/s, lr: 7.0e-05 loss: 1.267e-01] jam: 4%|▍ | 819/20000 [03:58<1:20:21, 3.98it/s, lr: 7.0e-05 loss: 8.135e-02] jam: 4%|▍ | 820/20000 [03:58<1:44:11, 3.07it/s, lr: 7.0e-05 loss: 8.135e-02] jam: 4%|▍ | 820/20000 [03:59<1:44:11, 3.07it/s, lr: 7.0e-05 loss: 3.361e-02] jam: 4%|▍ | 821/20000 [03:59<1:47:55, 2.96it/s, lr: 7.0e-05 loss: 3.361e-02] jam: 4%|▍ | 821/20000 [03:59<1:47:55, 2.96it/s, lr: 7.0e-05 loss: 1.523e-01] jam: 4%|▍ | 822/20000 [03:59<1:19:24, 4.03it/s, lr: 7.0e-05 loss: 1.523e-01] jam: 4%|▍ | 822/20000 [03:59<1:19:24, 4.03it/s, lr: 7.0e-05 loss: 1.609e-01] jam: 4%|▍ | 823/20000 [03:59<1:11:30, 4.47it/s, lr: 7.0e-05 loss: 1.609e-01] jam: 4%|▍ | 823/20000 [04:00<1:11:30, 4.47it/s, lr: 7.0e-05 loss: 2.333e-01] jam: 4%|▍ | 824/20000 [04:00<1:38:21, 3.25it/s, lr: 7.0e-05 loss: 2.333e-01] jam: 4%|▍ | 824/20000 [04:00<1:38:21, 3.25it/s, lr: 7.0e-05 loss: 9.427e-02] jam: 4%|▍ | 825/20000 [04:00<1:46:51, 2.99it/s, lr: 7.0e-05 loss: 9.427e-02] jam: 4%|▍ | 825/20000 [04:00<1:46:51, 2.99it/s, lr: 7.0e-05 loss: 7.837e-02] | |
| Saving at step 825 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000825.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 826/20000 [04:01<1:46:51, 2.99it/s, lr: 7.0e-05 loss: 9.204e-02] jam: 4%|▍ | 827/20000 [04:01<1:23:41, 3.82it/s, lr: 7.0e-05 loss: 9.204e-02] jam: 4%|▍ | 827/20000 [04:01<1:23:41, 3.82it/s, lr: 7.0e-05 loss: 1.489e-01] jam: 4%|▍ | 828/20000 [04:01<1:09:01, 4.63it/s, lr: 7.0e-05 loss: 1.489e-01] jam: 4%|▍ | 828/20000 [04:01<1:09:01, 4.63it/s, lr: 7.0e-05 loss: 2.202e-01] jam: 4%|▍ | 829/20000 [04:01<1:36:52, 3.30it/s, lr: 7.0e-05 loss: 2.202e-01] jam: 4%|▍ | 829/20000 [04:02<1:36:52, 3.30it/s, lr: 7.0e-05 loss: 1.836e-01] jam: 4%|▍ | 830/20000 [04:02<1:46:32, 3.00it/s, lr: 7.0e-05 loss: 1.836e-01] jam: 4%|▍ | 830/20000 [04:02<1:46:32, 3.00it/s, lr: 7.0e-05 loss: 1.118e-01] jam: 4%|▍ | 831/20000 [04:02<1:19:48, 4.00it/s, lr: 7.0e-05 loss: 1.118e-01] jam: 4%|▍ | 831/20000 [04:02<1:19:48, 4.00it/s, lr: 7.0e-05 loss: 1.413e-01] jam: 4%|▍ | 832/20000 [04:02<1:07:18, 4.75it/s, lr: 7.0e-05 loss: 1.413e-01] jam: 4%|▍ | 832/20000 [04:02<1:07:18, 4.75it/s, lr: 7.0e-05 loss: 1.137e-01] jam: 4%|▍ | 833/20000 [04:02<1:05:58, 4.84it/s, lr: 7.0e-05 loss: 1.137e-01] jam: 4%|▍ | 833/20000 [04:02<1:05:58, 4.84it/s, lr: 7.0e-05 loss: 1.911e-01] jam: 4%|▍ | 834/20000 [04:02<1:05:45, 4.86it/s, lr: 7.0e-05 loss: 1.911e-01] jam: 4%|▍ | 834/20000 [04:03<1:05:45, 4.86it/s, lr: 7.0e-05 loss: 1.171e-01] jam: 4%|▍ | 835/20000 [04:03<1:04:55, 4.92it/s, lr: 7.0e-05 loss: 1.171e-01] jam: 4%|▍ | 835/20000 [04:03<1:04:55, 4.92it/s, lr: 7.0e-05 loss: 1.383e-01] jam: 4%|▍ | 836/20000 [04:03<1:39:27, 3.21it/s, lr: 7.0e-05 loss: 1.383e-01] jam: 4%|▍ | 836/20000 [04:03<1:39:27, 3.21it/s, lr: 7.0e-05 loss: 1.116e-01] jam: 4%|▍ | 837/20000 [04:03<1:23:50, 3.81it/s, lr: 7.0e-05 loss: 1.116e-01] jam: 4%|▍ | 837/20000 [04:04<1:23:50, 3.81it/s, lr: 7.0e-05 loss: 1.853e-01] jam: 4%|▍ | 838/20000 [04:04<1:41:07, 3.16it/s, lr: 7.0e-05 loss: 1.853e-01] jam: 4%|▍ | 838/20000 [04:04<1:41:07, 3.16it/s, lr: 7.0e-05 loss: 1.730e-01] jam: 4%|▍ | 839/20000 [04:04<1:09:12, 4.61it/s, lr: 7.0e-05 loss: 1.730e-01] jam: 4%|▍ | 839/20000 [04:04<1:09:12, 4.61it/s, lr: 7.0e-05 loss: 1.254e-01] jam: 4%|▍ | 840/20000 [04:04<1:37:24, 3.28it/s, lr: 7.0e-05 loss: 1.254e-01] jam: 4%|▍ | 840/20000 [04:04<1:37:24, 3.28it/s, lr: 7.0e-05 loss: 7.758e-02] jam: 4%|▍ | 841/20000 [04:04<1:53:57, 2.80it/s, lr: 7.0e-05 loss: 7.758e-02] jam: 4%|▍ | 841/20000 [04:05<1:53:57, 2.80it/s, lr: 7.0e-05 loss: 1.209e-01] jam: 4%|▍ | 842/20000 [04:05<1:15:20, 4.24it/s, lr: 7.0e-05 loss: 1.209e-01] jam: 4%|▍ | 842/20000 [04:05<1:15:20, 4.24it/s, lr: 7.0e-05 loss: 1.121e-01] jam: 4%|▍ | 843/20000 [04:05<1:06:11, 4.82it/s, lr: 7.0e-05 loss: 1.121e-01] jam: 4%|▍ | 843/20000 [04:05<1:06:11, 4.82it/s, lr: 7.0e-05 loss: 1.066e-01] jam: 4%|▍ | 844/20000 [04:05<59:58, 5.32it/s, lr: 7.0e-05 loss: 1.066e-01] jam: 4%|▍ | 844/20000 [04:05<59:58, 5.32it/s, lr: 7.0e-05 loss: 1.163e-01] jam: 4%|▍ | 845/20000 [04:05<1:02:25, 5.11it/s, lr: 7.0e-05 loss: 1.163e-01] jam: 4%|▍ | 845/20000 [04:06<1:02:25, 5.11it/s, lr: 7.0e-05 loss: 2.782e-01] jam: 4%|▍ | 846/20000 [04:06<1:37:53, 3.26it/s, lr: 7.0e-05 loss: 2.782e-01] jam: 4%|▍ | 846/20000 [04:06<1:37:53, 3.26it/s, lr: 7.0e-05 loss: 1.412e-01] jam: 4%|▍ | 847/20000 [04:06<1:07:20, 4.74it/s, lr: 7.0e-05 loss: 1.412e-01] jam: 4%|▍ | 847/20000 [04:06<1:07:20, 4.74it/s, lr: 7.0e-05 loss: 7.560e-02] jam: 4%|▍ | 848/20000 [04:06<1:35:36, 3.34it/s, lr: 7.0e-05 loss: 7.560e-02] jam: 4%|▍ | 848/20000 [04:06<1:35:36, 3.34it/s, lr: 7.0e-05 loss: 1.087e-01] jam: 4%|▍ | 849/20000 [04:06<1:44:41, 3.05it/s, lr: 7.0e-05 loss: 1.087e-01] jam: 4%|▍ | 849/20000 [04:07<1:44:41, 3.05it/s, lr: 7.0e-05 loss: 2.340e-01] jam: 4%|▍ | 850/20000 [04:07<1:17:04, 4.14it/s, lr: 7.0e-05 loss: 2.340e-01] jam: 4%|▍ | 850/20000 [04:07<1:17:04, 4.14it/s, lr: 7.0e-05 loss: 1.119e-01] | |
| Saving at step 850 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000850.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 851/20000 [04:07<1:17:04, 4.14it/s, lr: 7.0e-05 loss: 8.443e-02] jam: 4%|▍ | 852/20000 [04:07<1:13:33, 4.34it/s, lr: 7.0e-05 loss: 8.443e-02] jam: 4%|▍ | 852/20000 [04:08<1:13:33, 4.34it/s, lr: 7.0e-05 loss: 1.207e-01] jam: 4%|▍ | 853/20000 [04:08<1:11:21, 4.47it/s, lr: 7.0e-05 loss: 1.207e-01] jam: 4%|▍ | 853/20000 [04:08<1:11:21, 4.47it/s, lr: 7.0e-05 loss: 1.989e-01] jam: 4%|▍ | 854/20000 [04:08<1:07:24, 4.73it/s, lr: 7.0e-05 loss: 1.989e-01] jam: 4%|▍ | 854/20000 [04:08<1:07:24, 4.73it/s, lr: 7.0e-05 loss: 2.244e-01] jam: 4%|▍ | 855/20000 [04:08<1:01:11, 5.21it/s, lr: 7.0e-05 loss: 2.244e-01] jam: 4%|▍ | 855/20000 [04:08<1:01:11, 5.21it/s, lr: 7.0e-05 loss: 1.666e-01] jam: 4%|▍ | 856/20000 [04:08<1:04:14, 4.97it/s, lr: 7.0e-05 loss: 1.666e-01] jam: 4%|▍ | 856/20000 [04:09<1:04:14, 4.97it/s, lr: 7.0e-05 loss: 3.413e-01] jam: 4%|▍ | 857/20000 [04:09<1:35:48, 3.33it/s, lr: 7.0e-05 loss: 3.413e-01] jam: 4%|▍ | 857/20000 [04:09<1:35:48, 3.33it/s, lr: 7.0e-05 loss: 1.366e-01] jam: 4%|▍ | 858/20000 [04:09<1:44:23, 3.06it/s, lr: 7.0e-05 loss: 1.366e-01] jam: 4%|▍ | 858/20000 [04:09<1:44:23, 3.06it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▍ | 859/20000 [04:09<1:19:19, 4.02it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▍ | 859/20000 [04:10<1:19:19, 4.02it/s, lr: 7.0e-05 loss: 1.349e-01] jam: 4%|▍ | 860/20000 [04:10<1:44:49, 3.04it/s, lr: 7.0e-05 loss: 1.349e-01] jam: 4%|▍ | 860/20000 [04:10<1:44:49, 3.04it/s, lr: 7.0e-05 loss: 7.936e-02] jam: 4%|▍ | 861/20000 [04:10<1:47:33, 2.97it/s, lr: 7.0e-05 loss: 7.936e-02] jam: 4%|▍ | 861/20000 [04:10<1:47:33, 2.97it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 4%|▍ | 862/20000 [04:10<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 4%|▍ | 862/20000 [04:11<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 4%|▍ | 863/20000 [04:11<1:49:19, 2.92it/s, lr: 7.0e-05 loss: 1.626e-01] jam: 4%|▍ | 863/20000 [04:11<1:49:19, 2.92it/s, lr: 7.0e-05 loss: 9.881e-02] jam: 4%|▍ | 864/20000 [04:11<1:21:40, 3.91it/s, lr: 7.0e-05 loss: 9.881e-02] jam: 4%|▍ | 864/20000 [04:11<1:21:40, 3.91it/s, lr: 7.0e-05 loss: 1.764e-01] jam: 4%|▍ | 865/20000 [04:11<1:11:15, 4.48it/s, lr: 7.0e-05 loss: 1.764e-01] jam: 4%|▍ | 865/20000 [04:11<1:11:15, 4.48it/s, lr: 7.0e-05 loss: 1.498e-01] jam: 4%|▍ | 866/20000 [04:11<1:06:00, 4.83it/s, lr: 7.0e-05 loss: 1.498e-01] jam: 4%|▍ | 866/20000 [04:11<1:06:00, 4.83it/s, lr: 7.0e-05 loss: 2.082e-01] jam: 4%|▍ | 867/20000 [04:11<1:03:14, 5.04it/s, lr: 7.0e-05 loss: 2.082e-01] jam: 4%|▍ | 867/20000 [04:12<1:03:14, 5.04it/s, lr: 7.0e-05 loss: 1.122e-01] jam: 4%|▍ | 868/20000 [04:12<1:36:11, 3.31it/s, lr: 7.0e-05 loss: 1.122e-01] jam: 4%|▍ | 868/20000 [04:12<1:36:11, 3.31it/s, lr: 7.0e-05 loss: 1.925e-01] jam: 4%|▍ | 869/20000 [04:12<1:25:57, 3.71it/s, lr: 7.0e-05 loss: 1.925e-01] jam: 4%|▍ | 869/20000 [04:12<1:25:57, 3.71it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▍ | 870/20000 [04:12<1:03:44, 5.00it/s, lr: 7.0e-05 loss: 1.291e-01] jam: 4%|▍ | 870/20000 [04:12<1:03:44, 5.00it/s, lr: 7.0e-05 loss: 1.659e-01] jam: 4%|▍ | 871/20000 [04:12<1:37:43, 3.26it/s, lr: 7.0e-05 loss: 1.659e-01] jam: 4%|▍ | 871/20000 [04:13<1:37:43, 3.26it/s, lr: 7.0e-05 loss: 2.175e-01] jam: 4%|▍ | 872/20000 [04:13<1:11:25, 4.46it/s, lr: 7.0e-05 loss: 2.175e-01] jam: 4%|▍ | 872/20000 [04:13<1:11:25, 4.46it/s, lr: 7.0e-05 loss: 6.248e-02] jam: 4%|▍ | 873/20000 [04:13<1:37:07, 3.28it/s, lr: 7.0e-05 loss: 6.248e-02] jam: 4%|▍ | 873/20000 [04:13<1:37:07, 3.28it/s, lr: 7.0e-05 loss: 9.837e-02] jam: 4%|▍ | 874/20000 [04:13<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 9.837e-02] jam: 4%|▍ | 874/20000 [04:14<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 1.333e-01] jam: 4%|▍ | 875/20000 [04:14<1:14:15, 4.29it/s, lr: 7.0e-05 loss: 1.333e-01] jam: 4%|▍ | 875/20000 [04:14<1:14:15, 4.29it/s, lr: 7.0e-05 loss: 1.654e-01] | |
| Saving at step 875 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000875.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 4%|▍ | 876/20000 [04:14<1:14:15, 4.29it/s, lr: 7.0e-05 loss: 1.539e-01] jam: 4%|▍ | 877/20000 [04:14<1:36:06, 3.32it/s, lr: 7.0e-05 loss: 1.539e-01] jam: 4%|▍ | 877/20000 [04:15<1:36:06, 3.32it/s, lr: 7.0e-05 loss: 2.247e-01] jam: 4%|▍ | 878/20000 [04:15<1:22:02, 3.88it/s, lr: 7.0e-05 loss: 2.247e-01] jam: 4%|▍ | 878/20000 [04:15<1:22:02, 3.88it/s, lr: 7.0e-05 loss: 2.938e-01] jam: 4%|▍ | 879/20000 [04:15<1:38:55, 3.22it/s, lr: 7.0e-05 loss: 2.938e-01] jam: 4%|▍ | 879/20000 [04:15<1:38:55, 3.22it/s, lr: 7.0e-05 loss: 3.492e-01] jam: 4%|▍ | 880/20000 [04:15<1:21:18, 3.92it/s, lr: 7.0e-05 loss: 3.492e-01] jam: 4%|▍ | 880/20000 [04:15<1:21:18, 3.92it/s, lr: 7.0e-05 loss: 1.322e-01] jam: 4%|▍ | 881/20000 [04:15<1:11:07, 4.48it/s, lr: 7.0e-05 loss: 1.322e-01] jam: 4%|▍ | 881/20000 [04:16<1:11:07, 4.48it/s, lr: 7.0e-05 loss: 1.118e-01] jam: 4%|▍ | 882/20000 [04:16<1:42:49, 3.10it/s, lr: 7.0e-05 loss: 1.118e-01] jam: 4%|▍ | 882/20000 [04:16<1:42:49, 3.10it/s, lr: 7.0e-05 loss: 1.422e-01] jam: 4%|▍ | 883/20000 [04:16<1:17:58, 4.09it/s, lr: 7.0e-05 loss: 1.422e-01] jam: 4%|▍ | 883/20000 [04:16<1:17:58, 4.09it/s, lr: 7.0e-05 loss: 1.737e-01] jam: 4%|▍ | 884/20000 [04:16<1:08:49, 4.63it/s, lr: 7.0e-05 loss: 1.737e-01] jam: 4%|▍ | 884/20000 [04:17<1:08:49, 4.63it/s, lr: 7.0e-05 loss: 2.694e-01] jam: 4%|▍ | 885/20000 [04:17<1:40:25, 3.17it/s, lr: 7.0e-05 loss: 2.694e-01] jam: 4%|▍ | 885/20000 [04:17<1:40:25, 3.17it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 4%|▍ | 886/20000 [04:17<1:11:36, 4.45it/s, lr: 7.0e-05 loss: 1.315e-01] jam: 4%|▍ | 886/20000 [04:17<1:11:36, 4.45it/s, lr: 7.0e-05 loss: 8.670e-02] jam: 4%|▍ | 887/20000 [04:17<1:08:18, 4.66it/s, lr: 7.0e-05 loss: 8.670e-02] jam: 4%|▍ | 887/20000 [04:17<1:08:18, 4.66it/s, lr: 7.0e-05 loss: 1.070e-01] jam: 4%|▍ | 888/20000 [04:17<1:35:02, 3.35it/s, lr: 7.0e-05 loss: 1.070e-01] jam: 4%|▍ | 888/20000 [04:18<1:35:02, 3.35it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 4%|▍ | 889/20000 [04:18<1:18:25, 4.06it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 4%|▍ | 889/20000 [04:18<1:18:25, 4.06it/s, lr: 7.0e-05 loss: 9.969e-02] jam: 4%|▍ | 890/20000 [04:18<1:44:48, 3.04it/s, lr: 7.0e-05 loss: 9.969e-02] jam: 4%|▍ | 890/20000 [04:18<1:44:48, 3.04it/s, lr: 7.0e-05 loss: 1.156e-01] jam: 4%|▍ | 891/20000 [04:18<1:47:30, 2.96it/s, lr: 7.0e-05 loss: 1.156e-01] jam: 4%|▍ | 891/20000 [04:19<1:47:30, 2.96it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 4%|▍ | 892/20000 [04:19<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 4%|▍ | 892/20000 [04:19<1:51:48, 2.85it/s, lr: 7.0e-05 loss: 1.316e-01] jam: 4%|▍ | 893/20000 [04:19<1:23:56, 3.79it/s, lr: 7.0e-05 loss: 1.316e-01] jam: 4%|▍ | 893/20000 [04:19<1:23:56, 3.79it/s, lr: 7.0e-05 loss: 1.918e-01] jam: 4%|▍ | 894/20000 [04:19<1:21:46, 3.89it/s, lr: 7.0e-05 loss: 1.918e-01] jam: 4%|▍ | 894/20000 [04:19<1:21:46, 3.89it/s, lr: 7.0e-05 loss: 8.585e-02] jam: 4%|▍ | 895/20000 [04:19<1:06:09, 4.81it/s, lr: 7.0e-05 loss: 8.585e-02] jam: 4%|▍ | 895/20000 [04:20<1:06:09, 4.81it/s, lr: 7.0e-05 loss: 2.170e-01] jam: 4%|▍ | 896/20000 [04:20<1:33:58, 3.39it/s, lr: 7.0e-05 loss: 2.170e-01] jam: 4%|▍ | 896/20000 [04:20<1:33:58, 3.39it/s, lr: 7.0e-05 loss: 1.277e-01] jam: 4%|▍ | 897/20000 [04:20<1:07:47, 4.70it/s, lr: 7.0e-05 loss: 1.277e-01] jam: 4%|▍ | 897/20000 [04:20<1:07:47, 4.70it/s, lr: 7.0e-05 loss: 1.421e-01] jam: 4%|▍ | 898/20000 [04:20<1:01:06, 5.21it/s, lr: 7.0e-05 loss: 1.421e-01] jam: 4%|▍ | 898/20000 [04:20<1:01:06, 5.21it/s, lr: 7.0e-05 loss: 1.404e-01] jam: 4%|▍ | 899/20000 [04:20<59:40, 5.34it/s, lr: 7.0e-05 loss: 1.404e-01] jam: 4%|▍ | 899/20000 [04:20<59:40, 5.34it/s, lr: 7.0e-05 loss: 2.367e-01] jam: 4%|▍ | 900/20000 [04:20<1:01:55, 5.14it/s, lr: 7.0e-05 loss: 2.367e-01] jam: 4%|▍ | 900/20000 [04:21<1:01:55, 5.14it/s, lr: 7.0e-05 loss: 1.084e-01] | |
| Saving at step 900 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000900.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▍ | 901/20000 [04:21<1:01:55, 5.14it/s, lr: 7.0e-05 loss: 1.824e-01] jam: 5%|▍ | 902/20000 [04:21<1:14:46, 4.26it/s, lr: 7.0e-05 loss: 1.824e-01] jam: 5%|▍ | 902/20000 [04:21<1:14:46, 4.26it/s, lr: 7.0e-05 loss: 1.736e-01] jam: 5%|▍ | 903/20000 [04:21<1:05:47, 4.84it/s, lr: 7.0e-05 loss: 1.736e-01] jam: 5%|▍ | 903/20000 [04:21<1:05:47, 4.84it/s, lr: 7.0e-05 loss: 8.951e-02] jam: 5%|▍ | 904/20000 [04:21<1:04:55, 4.90it/s, lr: 7.0e-05 loss: 8.951e-02] jam: 5%|▍ | 904/20000 [04:22<1:04:55, 4.90it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 5%|▍ | 905/20000 [04:22<1:03:24, 5.02it/s, lr: 7.0e-05 loss: 1.140e-01] jam: 5%|▍ | 905/20000 [04:22<1:03:24, 5.02it/s, lr: 7.0e-05 loss: 1.051e-01] jam: 5%|▍ | 906/20000 [04:22<1:36:50, 3.29it/s, lr: 7.0e-05 loss: 1.051e-01] jam: 5%|▍ | 906/20000 [04:22<1:36:50, 3.29it/s, lr: 7.0e-05 loss: 1.724e-01] jam: 5%|▍ | 907/20000 [04:22<1:07:15, 4.73it/s, lr: 7.0e-05 loss: 1.724e-01] jam: 5%|▍ | 907/20000 [04:22<1:07:15, 4.73it/s, lr: 7.0e-05 loss: 4.126e-02] jam: 5%|▍ | 908/20000 [04:22<1:36:34, 3.30it/s, lr: 7.0e-05 loss: 4.126e-02] jam: 5%|▍ | 908/20000 [04:23<1:36:34, 3.30it/s, lr: 7.0e-05 loss: 1.591e-01] jam: 5%|▍ | 909/20000 [04:23<1:19:37, 4.00it/s, lr: 7.0e-05 loss: 1.591e-01] jam: 5%|▍ | 909/20000 [04:23<1:19:37, 4.00it/s, lr: 7.0e-05 loss: 9.731e-02] jam: 5%|▍ | 910/20000 [04:23<1:43:54, 3.06it/s, lr: 7.0e-05 loss: 9.731e-02] jam: 5%|▍ | 910/20000 [04:23<1:43:54, 3.06it/s, lr: 7.0e-05 loss: 2.166e-01] jam: 5%|▍ | 911/20000 [04:23<1:46:17, 2.99it/s, lr: 7.0e-05 loss: 2.166e-01] jam: 5%|▍ | 911/20000 [04:24<1:46:17, 2.99it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 5%|▍ | 912/20000 [04:24<1:56:15, 2.74it/s, lr: 7.0e-05 loss: 1.352e-01] jam: 5%|▍ | 912/20000 [04:24<1:56:15, 2.74it/s, lr: 7.0e-05 loss: 1.915e-01] jam: 5%|▍ | 913/20000 [04:24<1:27:02, 3.65it/s, lr: 7.0e-05 loss: 1.915e-01] jam: 5%|▍ | 913/20000 [04:24<1:27:02, 3.65it/s, lr: 7.0e-05 loss: 1.031e-01] jam: 5%|▍ | 914/20000 [04:24<1:24:48, 3.75it/s, lr: 7.0e-05 loss: 1.031e-01] jam: 5%|▍ | 914/20000 [04:24<1:24:48, 3.75it/s, lr: 7.0e-05 loss: 2.155e-01] jam: 5%|▍ | 915/20000 [04:24<1:13:39, 4.32it/s, lr: 7.0e-05 loss: 2.155e-01] jam: 5%|▍ | 915/20000 [04:25<1:13:39, 4.32it/s, lr: 7.0e-05 loss: 8.225e-02] jam: 5%|▍ | 916/20000 [04:25<1:41:42, 3.13it/s, lr: 7.0e-05 loss: 8.225e-02] jam: 5%|▍ | 916/20000 [04:25<1:41:42, 3.13it/s, lr: 7.0e-05 loss: 1.905e-01] jam: 5%|▍ | 917/20000 [04:25<1:12:22, 4.39it/s, lr: 7.0e-05 loss: 1.905e-01] jam: 5%|▍ | 917/20000 [04:25<1:12:22, 4.39it/s, lr: 7.0e-05 loss: 1.858e-01] jam: 5%|▍ | 918/20000 [04:25<1:36:18, 3.30it/s, lr: 7.0e-05 loss: 1.858e-01] jam: 5%|▍ | 918/20000 [04:26<1:36:18, 3.30it/s, lr: 7.0e-05 loss: 1.100e-01] jam: 5%|▍ | 919/20000 [04:26<1:52:17, 2.83it/s, lr: 7.0e-05 loss: 1.100e-01] jam: 5%|▍ | 919/20000 [04:26<1:52:17, 2.83it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 5%|▍ | 920/20000 [04:26<1:48:55, 2.92it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 5%|▍ | 920/20000 [04:26<1:48:55, 2.92it/s, lr: 7.0e-05 loss: 6.970e-02] jam: 5%|▍ | 921/20000 [04:26<1:54:08, 2.79it/s, lr: 7.0e-05 loss: 6.970e-02] jam: 5%|▍ | 921/20000 [04:27<1:54:08, 2.79it/s, lr: 7.0e-05 loss: 1.685e-01] jam: 5%|▍ | 922/20000 [04:27<1:24:55, 3.74it/s, lr: 7.0e-05 loss: 1.685e-01] jam: 5%|▍ | 922/20000 [04:27<1:24:55, 3.74it/s, lr: 7.0e-05 loss: 1.058e-01] jam: 5%|▍ | 923/20000 [04:27<1:47:44, 2.95it/s, lr: 7.0e-05 loss: 1.058e-01] jam: 5%|▍ | 923/20000 [04:27<1:47:44, 2.95it/s, lr: 7.0e-05 loss: 2.318e-01] jam: 5%|▍ | 924/20000 [04:27<1:12:37, 4.38it/s, lr: 7.0e-05 loss: 2.318e-01] jam: 5%|▍ | 924/20000 [04:28<1:12:37, 4.38it/s, lr: 7.0e-05 loss: 1.209e-01] jam: 5%|▍ | 925/20000 [04:28<1:38:16, 3.23it/s, lr: 7.0e-05 loss: 1.209e-01] jam: 5%|▍ | 925/20000 [04:28<1:38:16, 3.23it/s, lr: 7.0e-05 loss: 3.327e-01] | |
| Saving at step 925 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000925.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▍ | 926/20000 [04:28<1:38:16, 3.23it/s, lr: 7.0e-05 loss: 3.139e-01] jam: 5%|▍ | 927/20000 [04:28<1:40:09, 3.17it/s, lr: 7.0e-05 loss: 3.139e-01] jam: 5%|▍ | 927/20000 [04:29<1:40:09, 3.17it/s, lr: 7.0e-05 loss: 1.382e-01] jam: 5%|▍ | 928/20000 [04:29<1:44:41, 3.04it/s, lr: 7.0e-05 loss: 1.382e-01] jam: 5%|▍ | 928/20000 [04:29<1:44:41, 3.04it/s, lr: 7.0e-05 loss: 1.382e-01] jam: 5%|▍ | 929/20000 [04:29<1:47:30, 2.96it/s, lr: 7.0e-05 loss: 1.382e-01] jam: 5%|▍ | 929/20000 [04:29<1:47:30, 2.96it/s, lr: 7.0e-05 loss: 1.229e-01] jam: 5%|▍ | 930/20000 [04:29<1:12:39, 4.37it/s, lr: 7.0e-05 loss: 1.229e-01] jam: 5%|▍ | 930/20000 [04:29<1:12:39, 4.37it/s, lr: 7.0e-05 loss: 1.188e-01] jam: 5%|▍ | 931/20000 [04:29<1:02:49, 5.06it/s, lr: 7.0e-05 loss: 1.188e-01] jam: 5%|▍ | 931/20000 [04:30<1:02:49, 5.06it/s, lr: 7.0e-05 loss: 1.671e-01] jam: 5%|▍ | 932/20000 [04:30<1:40:15, 3.17it/s, lr: 7.0e-05 loss: 1.671e-01] jam: 5%|▍ | 932/20000 [04:30<1:40:15, 3.17it/s, lr: 7.0e-05 loss: 1.105e-01] jam: 5%|▍ | 933/20000 [04:30<1:17:40, 4.09it/s, lr: 7.0e-05 loss: 1.105e-01] jam: 5%|▍ | 933/20000 [04:30<1:17:40, 4.09it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 5%|▍ | 934/20000 [04:30<1:08:14, 4.66it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 5%|▍ | 934/20000 [04:30<1:08:14, 4.66it/s, lr: 7.0e-05 loss: 2.399e-01] jam: 5%|▍ | 935/20000 [04:30<57:12, 5.56it/s, lr: 7.0e-05 loss: 2.399e-01] jam: 5%|▍ | 935/20000 [04:31<57:12, 5.56it/s, lr: 7.0e-05 loss: 1.429e-01] jam: 5%|▍ | 936/20000 [04:31<55:02, 5.77it/s, lr: 7.0e-05 loss: 1.429e-01] jam: 5%|▍ | 936/20000 [04:31<55:02, 5.77it/s, lr: 7.0e-05 loss: 1.726e-01] jam: 5%|▍ | 937/20000 [04:31<1:33:18, 3.41it/s, lr: 7.0e-05 loss: 1.726e-01] jam: 5%|▍ | 937/20000 [04:31<1:33:18, 3.41it/s, lr: 7.0e-05 loss: 2.788e-01] jam: 5%|▍ | 938/20000 [04:31<1:51:41, 2.84it/s, lr: 7.0e-05 loss: 2.788e-01] jam: 5%|▍ | 938/20000 [04:31<1:51:41, 2.84it/s, lr: 7.0e-05 loss: 2.223e-01] jam: 5%|▍ | 939/20000 [04:31<1:13:45, 4.31it/s, lr: 7.0e-05 loss: 2.223e-01] jam: 5%|▍ | 939/20000 [04:32<1:13:45, 4.31it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 5%|▍ | 940/20000 [04:32<1:38:05, 3.24it/s, lr: 7.0e-05 loss: 1.774e-01] jam: 5%|▍ | 940/20000 [04:32<1:38:05, 3.24it/s, lr: 7.0e-05 loss: 2.515e-01] jam: 5%|▍ | 941/20000 [04:32<1:17:32, 4.10it/s, lr: 7.0e-05 loss: 2.515e-01] jam: 5%|▍ | 941/20000 [04:32<1:17:32, 4.10it/s, lr: 7.0e-05 loss: 9.471e-02] jam: 5%|▍ | 942/20000 [04:32<1:45:53, 3.00it/s, lr: 7.0e-05 loss: 9.471e-02] jam: 5%|▍ | 942/20000 [04:33<1:45:53, 3.00it/s, lr: 7.0e-05 loss: 1.417e-01] jam: 5%|▍ | 943/20000 [04:33<1:47:15, 2.96it/s, lr: 7.0e-05 loss: 1.417e-01] jam: 5%|▍ | 943/20000 [04:33<1:47:15, 2.96it/s, lr: 7.0e-05 loss: 1.276e-01] jam: 5%|▍ | 944/20000 [04:33<1:21:32, 3.89it/s, lr: 7.0e-05 loss: 1.276e-01] jam: 5%|▍ | 944/20000 [04:33<1:21:32, 3.89it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 5%|▍ | 945/20000 [04:33<1:08:38, 4.63it/s, lr: 7.0e-05 loss: 1.181e-01] jam: 5%|▍ | 945/20000 [04:33<1:08:38, 4.63it/s, lr: 7.0e-05 loss: 1.107e-01] jam: 5%|▍ | 946/20000 [04:33<1:06:26, 4.78it/s, lr: 7.0e-05 loss: 1.107e-01] jam: 5%|▍ | 946/20000 [04:34<1:06:26, 4.78it/s, lr: 7.0e-05 loss: 2.553e-01] jam: 5%|▍ | 947/20000 [04:34<1:10:30, 4.50it/s, lr: 7.0e-05 loss: 2.553e-01] jam: 5%|▍ | 947/20000 [04:34<1:10:30, 4.50it/s, lr: 7.0e-05 loss: 1.256e-01] jam: 5%|▍ | 948/20000 [04:34<1:36:20, 3.30it/s, lr: 7.0e-05 loss: 1.256e-01] jam: 5%|▍ | 948/20000 [04:34<1:36:20, 3.30it/s, lr: 7.0e-05 loss: 1.344e-01] jam: 5%|▍ | 949/20000 [04:34<1:47:52, 2.94it/s, lr: 7.0e-05 loss: 1.344e-01] jam: 5%|▍ | 949/20000 [04:35<1:47:52, 2.94it/s, lr: 7.0e-05 loss: 1.551e-01] jam: 5%|▍ | 950/20000 [04:35<1:47:44, 2.95it/s, lr: 7.0e-05 loss: 1.551e-01] jam: 5%|▍ | 950/20000 [04:35<1:47:44, 2.95it/s, lr: 7.0e-05 loss: 2.749e-01] | |
| Saving at step 950 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000950.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▍ | 951/20000 [04:36<1:47:44, 2.95it/s, lr: 7.0e-05 loss: 8.140e-02] jam: 5%|▍ | 952/20000 [04:36<1:42:11, 3.11it/s, lr: 7.0e-05 loss: 8.140e-02] jam: 5%|▍ | 952/20000 [04:36<1:42:11, 3.11it/s, lr: 7.0e-05 loss: 9.917e-02] jam: 5%|▍ | 953/20000 [04:36<1:12:47, 4.36it/s, lr: 7.0e-05 loss: 9.917e-02] jam: 5%|▍ | 953/20000 [04:36<1:12:47, 4.36it/s, lr: 7.0e-05 loss: 1.641e-01] jam: 5%|▍ | 954/20000 [04:36<1:35:27, 3.33it/s, lr: 7.0e-05 loss: 1.641e-01] jam: 5%|▍ | 954/20000 [04:36<1:35:27, 3.33it/s, lr: 7.0e-05 loss: 9.614e-02] jam: 5%|▍ | 955/20000 [04:36<1:50:59, 2.86it/s, lr: 7.0e-05 loss: 9.614e-02] jam: 5%|▍ | 955/20000 [04:37<1:50:59, 2.86it/s, lr: 7.0e-05 loss: 6.968e-02] jam: 5%|▍ | 956/20000 [04:37<1:13:45, 4.30it/s, lr: 7.0e-05 loss: 6.968e-02] jam: 5%|▍ | 956/20000 [04:37<1:13:45, 4.30it/s, lr: 7.0e-05 loss: 6.179e-02] jam: 5%|▍ | 957/20000 [04:37<1:06:34, 4.77it/s, lr: 7.0e-05 loss: 6.179e-02] jam: 5%|▍ | 957/20000 [04:37<1:06:34, 4.77it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 5%|▍ | 958/20000 [04:37<1:36:02, 3.30it/s, lr: 7.0e-05 loss: 1.164e-01] jam: 5%|▍ | 958/20000 [04:37<1:36:02, 3.30it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 5%|▍ | 959/20000 [04:37<1:20:05, 3.96it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 5%|▍ | 959/20000 [04:38<1:20:05, 3.96it/s, lr: 7.0e-05 loss: 7.870e-02] jam: 5%|▍ | 960/20000 [04:38<1:10:46, 4.48it/s, lr: 7.0e-05 loss: 7.870e-02] jam: 5%|▍ | 960/20000 [04:38<1:10:46, 4.48it/s, lr: 7.0e-05 loss: 1.434e-01] jam: 5%|▍ | 961/20000 [04:38<1:06:33, 4.77it/s, lr: 7.0e-05 loss: 1.434e-01] jam: 5%|▍ | 961/20000 [04:38<1:06:33, 4.77it/s, lr: 7.0e-05 loss: 1.712e-01] jam: 5%|▍ | 962/20000 [04:38<1:10:29, 4.50it/s, lr: 7.0e-05 loss: 1.712e-01] jam: 5%|▍ | 962/20000 [04:38<1:10:29, 4.50it/s, lr: 7.0e-05 loss: 9.335e-02] jam: 5%|▍ | 963/20000 [04:38<1:04:51, 4.89it/s, lr: 7.0e-05 loss: 9.335e-02] jam: 5%|▍ | 963/20000 [04:38<1:04:51, 4.89it/s, lr: 7.0e-05 loss: 2.114e-01] jam: 5%|▍ | 964/20000 [04:38<1:05:27, 4.85it/s, lr: 7.0e-05 loss: 2.114e-01] jam: 5%|▍ | 964/20000 [04:39<1:05:27, 4.85it/s, lr: 7.0e-05 loss: 1.995e-01] jam: 5%|▍ | 965/20000 [04:39<1:03:47, 4.97it/s, lr: 7.0e-05 loss: 1.995e-01] jam: 5%|▍ | 965/20000 [04:39<1:03:47, 4.97it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 5%|▍ | 966/20000 [04:39<1:36:04, 3.30it/s, lr: 7.0e-05 loss: 1.249e-01] jam: 5%|▍ | 966/20000 [04:39<1:36:04, 3.30it/s, lr: 7.0e-05 loss: 1.317e-01] jam: 5%|▍ | 967/20000 [04:39<1:47:17, 2.96it/s, lr: 7.0e-05 loss: 1.317e-01] jam: 5%|▍ | 967/20000 [04:40<1:47:17, 2.96it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 5%|▍ | 968/20000 [04:40<1:15:05, 4.22it/s, lr: 7.0e-05 loss: 1.228e-01] jam: 5%|▍ | 968/20000 [04:40<1:15:05, 4.22it/s, lr: 7.0e-05 loss: 1.403e-01] jam: 5%|▍ | 969/20000 [04:40<1:37:40, 3.25it/s, lr: 7.0e-05 loss: 1.403e-01] jam: 5%|▍ | 969/20000 [04:40<1:37:40, 3.25it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 5%|▍ | 970/20000 [04:40<1:17:05, 4.11it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 5%|▍ | 970/20000 [04:40<1:17:05, 4.11it/s, lr: 7.0e-05 loss: 9.502e-02] jam: 5%|▍ | 971/20000 [04:40<1:41:53, 3.11it/s, lr: 7.0e-05 loss: 9.502e-02] jam: 5%|▍ | 971/20000 [04:41<1:41:53, 3.11it/s, lr: 7.0e-05 loss: 1.642e-01] jam: 5%|▍ | 972/20000 [04:41<1:45:34, 3.00it/s, lr: 7.0e-05 loss: 1.642e-01] jam: 5%|▍ | 972/20000 [04:41<1:45:34, 3.00it/s, lr: 7.0e-05 loss: 1.257e-01] jam: 5%|▍ | 973/20000 [04:41<1:17:30, 4.09it/s, lr: 7.0e-05 loss: 1.257e-01] jam: 5%|▍ | 973/20000 [04:41<1:17:30, 4.09it/s, lr: 7.0e-05 loss: 1.618e-01] jam: 5%|▍ | 974/20000 [04:41<1:09:19, 4.57it/s, lr: 7.0e-05 loss: 1.618e-01] jam: 5%|▍ | 974/20000 [04:42<1:09:19, 4.57it/s, lr: 7.0e-05 loss: 7.403e-02] jam: 5%|▍ | 975/20000 [04:42<1:41:05, 3.14it/s, lr: 7.0e-05 loss: 7.403e-02] jam: 5%|▍ | 975/20000 [04:42<1:41:05, 3.14it/s, lr: 7.0e-05 loss: 6.142e-02] | |
| Saving at step 975 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000000975.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▍ | 976/20000 [04:42<1:41:05, 3.14it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 5%|▍ | 977/20000 [04:42<1:22:17, 3.85it/s, lr: 7.0e-05 loss: 1.071e-01] jam: 5%|▍ | 977/20000 [04:42<1:22:17, 3.85it/s, lr: 7.0e-05 loss: 1.812e-01] jam: 5%|▍ | 978/20000 [04:42<1:08:27, 4.63it/s, lr: 7.0e-05 loss: 1.812e-01] jam: 5%|▍ | 978/20000 [04:43<1:08:27, 4.63it/s, lr: 7.0e-05 loss: 1.214e-01] jam: 5%|▍ | 979/20000 [04:43<1:04:55, 4.88it/s, lr: 7.0e-05 loss: 1.214e-01] jam: 5%|▍ | 979/20000 [04:43<1:04:55, 4.88it/s, lr: 7.0e-05 loss: 1.947e-01] jam: 5%|▍ | 980/20000 [04:43<1:35:29, 3.32it/s, lr: 7.0e-05 loss: 1.947e-01] jam: 5%|▍ | 980/20000 [04:43<1:35:29, 3.32it/s, lr: 7.0e-05 loss: 1.010e-01] jam: 5%|▍ | 981/20000 [04:43<1:52:00, 2.83it/s, lr: 7.0e-05 loss: 1.010e-01] jam: 5%|▍ | 981/20000 [04:44<1:52:00, 2.83it/s, lr: 7.0e-05 loss: 1.694e-01] jam: 5%|▍ | 982/20000 [04:44<1:15:01, 4.22it/s, lr: 7.0e-05 loss: 1.694e-01] jam: 5%|▍ | 982/20000 [04:44<1:15:01, 4.22it/s, lr: 7.0e-05 loss: 1.206e-01] jam: 5%|▍ | 983/20000 [04:44<1:38:10, 3.23it/s, lr: 7.0e-05 loss: 1.206e-01] jam: 5%|▍ | 983/20000 [04:44<1:38:10, 3.23it/s, lr: 7.0e-05 loss: 2.400e-01] jam: 5%|▍ | 984/20000 [04:44<1:19:45, 3.97it/s, lr: 7.0e-05 loss: 2.400e-01] jam: 5%|▍ | 984/20000 [04:44<1:19:45, 3.97it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 5%|▍ | 985/20000 [04:44<1:17:58, 4.06it/s, lr: 7.0e-05 loss: 1.483e-01] jam: 5%|▍ | 985/20000 [04:45<1:17:58, 4.06it/s, lr: 7.0e-05 loss: 6.581e-02] jam: 5%|▍ | 986/20000 [04:45<1:38:46, 3.21it/s, lr: 7.0e-05 loss: 6.581e-02] jam: 5%|▍ | 986/20000 [04:45<1:38:46, 3.21it/s, lr: 7.0e-05 loss: 2.226e-01] jam: 5%|▍ | 987/20000 [04:45<1:49:10, 2.90it/s, lr: 7.0e-05 loss: 2.226e-01] jam: 5%|▍ | 987/20000 [04:45<1:49:10, 2.90it/s, lr: 7.0e-05 loss: 1.182e-01] jam: 5%|▍ | 988/20000 [04:45<1:14:08, 4.27it/s, lr: 7.0e-05 loss: 1.182e-01] jam: 5%|▍ | 988/20000 [04:45<1:14:08, 4.27it/s, lr: 7.0e-05 loss: 8.717e-02] jam: 5%|▍ | 989/20000 [04:45<1:04:51, 4.88it/s, lr: 7.0e-05 loss: 8.717e-02] jam: 5%|▍ | 989/20000 [04:46<1:04:51, 4.88it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 5%|▍ | 990/20000 [04:46<1:03:29, 4.99it/s, lr: 7.0e-05 loss: 1.347e-01] jam: 5%|▍ | 990/20000 [04:46<1:03:29, 4.99it/s, lr: 7.0e-05 loss: 1.485e-01] jam: 5%|▍ | 991/20000 [04:46<1:38:52, 3.20it/s, lr: 7.0e-05 loss: 1.485e-01] jam: 5%|▍ | 991/20000 [04:46<1:38:52, 3.20it/s, lr: 7.0e-05 loss: 1.312e-01] jam: 5%|▍ | 992/20000 [04:46<1:50:52, 2.86it/s, lr: 7.0e-05 loss: 1.312e-01] jam: 5%|▍ | 992/20000 [04:47<1:50:52, 2.86it/s, lr: 7.0e-05 loss: 1.819e-01] jam: 5%|▍ | 993/20000 [04:47<1:16:05, 4.16it/s, lr: 7.0e-05 loss: 1.819e-01] jam: 5%|▍ | 993/20000 [04:47<1:16:05, 4.16it/s, lr: 7.0e-05 loss: 8.577e-02] jam: 5%|▍ | 994/20000 [04:47<1:38:11, 3.23it/s, lr: 7.0e-05 loss: 8.577e-02] jam: 5%|▍ | 994/20000 [04:47<1:38:11, 3.23it/s, lr: 7.0e-05 loss: 5.129e-02] jam: 5%|▍ | 995/20000 [04:47<1:50:33, 2.86it/s, lr: 7.0e-05 loss: 5.129e-02] jam: 5%|▍ | 995/20000 [04:48<1:50:33, 2.86it/s, lr: 7.0e-05 loss: 1.001e-01] jam: 5%|▍ | 996/20000 [04:48<1:50:00, 2.88it/s, lr: 7.0e-05 loss: 1.001e-01] jam: 5%|▍ | 996/20000 [04:48<1:50:00, 2.88it/s, lr: 7.0e-05 loss: 6.789e-02] jam: 5%|▍ | 997/20000 [04:48<1:51:40, 2.84it/s, lr: 7.0e-05 loss: 6.789e-02] jam: 5%|▍ | 997/20000 [04:48<1:51:40, 2.84it/s, lr: 7.0e-05 loss: 1.688e-01] jam: 5%|▍ | 998/20000 [04:48<1:15:15, 4.21it/s, lr: 7.0e-05 loss: 1.688e-01] jam: 5%|▍ | 998/20000 [04:48<1:15:15, 4.21it/s, lr: 7.0e-05 loss: 7.343e-02] jam: 5%|▍ | 999/20000 [04:48<1:06:01, 4.80it/s, lr: 7.0e-05 loss: 7.343e-02] jam: 5%|▍ | 999/20000 [04:49<1:06:01, 4.80it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 5%|▌ | 1000/20000 [04:49<1:35:22, 3.32it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 5%|▌ | 1000/20000 [04:49<1:35:22, 3.32it/s, lr: 7.0e-05 loss: 3.502e-02] | |
| Saving at step 1000 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001000.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▌ | 1001/20000 [04:49<1:35:22, 3.32it/s, lr: 7.0e-05 loss: 2.077e-01] jam: 5%|▌ | 1002/20000 [04:49<1:16:06, 4.16it/s, lr: 7.0e-05 loss: 2.077e-01] jam: 5%|▌ | 1002/20000 [04:50<1:16:06, 4.16it/s, lr: 7.0e-05 loss: 9.167e-02] jam: 5%|▌ | 1003/20000 [04:50<1:21:28, 3.89it/s, lr: 7.0e-05 loss: 9.167e-02] jam: 5%|▌ | 1003/20000 [04:50<1:21:28, 3.89it/s, lr: 7.0e-05 loss: 1.715e-01] jam: 5%|▌ | 1004/20000 [04:50<1:46:46, 2.97it/s, lr: 7.0e-05 loss: 1.715e-01] jam: 5%|▌ | 1004/20000 [04:50<1:46:46, 2.97it/s, lr: 7.0e-05 loss: 7.874e-02] jam: 5%|▌ | 1005/20000 [04:50<1:47:23, 2.95it/s, lr: 7.0e-05 loss: 7.874e-02] jam: 5%|▌ | 1005/20000 [04:51<1:47:23, 2.95it/s, lr: 7.0e-05 loss: 1.385e-01] jam: 5%|▌ | 1006/20000 [04:51<1:15:04, 4.22it/s, lr: 7.0e-05 loss: 1.385e-01] jam: 5%|▌ | 1006/20000 [04:51<1:15:04, 4.22it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 5%|▌ | 1007/20000 [04:51<1:09:53, 4.53it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 5%|▌ | 1007/20000 [04:51<1:09:53, 4.53it/s, lr: 7.0e-05 loss: 1.639e-01] jam: 5%|▌ | 1008/20000 [04:51<1:39:02, 3.20it/s, lr: 7.0e-05 loss: 1.639e-01] jam: 5%|▌ | 1008/20000 [04:52<1:39:02, 3.20it/s, lr: 7.0e-05 loss: 1.179e-01] jam: 5%|▌ | 1009/20000 [04:52<1:46:02, 2.98it/s, lr: 7.0e-05 loss: 1.179e-01] jam: 5%|▌ | 1009/20000 [04:52<1:46:02, 2.98it/s, lr: 7.0e-05 loss: 1.805e-01] jam: 5%|▌ | 1010/20000 [04:52<1:05:16, 4.85it/s, lr: 7.0e-05 loss: 1.805e-01] jam: 5%|▌ | 1010/20000 [04:52<1:05:16, 4.85it/s, lr: 7.0e-05 loss: 1.392e-01] jam: 5%|▌ | 1011/20000 [04:52<1:03:55, 4.95it/s, lr: 7.0e-05 loss: 1.392e-01] jam: 5%|▌ | 1011/20000 [04:52<1:03:55, 4.95it/s, lr: 7.0e-05 loss: 7.758e-02] jam: 5%|▌ | 1012/20000 [04:52<1:34:18, 3.36it/s, lr: 7.0e-05 loss: 7.758e-02] jam: 5%|▌ | 1012/20000 [04:53<1:34:18, 3.36it/s, lr: 7.0e-05 loss: 2.432e-01] jam: 5%|▌ | 1013/20000 [04:53<1:48:51, 2.91it/s, lr: 7.0e-05 loss: 2.432e-01] jam: 5%|▌ | 1013/20000 [04:53<1:48:51, 2.91it/s, lr: 7.0e-05 loss: 5.762e-02] jam: 5%|▌ | 1014/20000 [04:53<1:25:29, 3.70it/s, lr: 7.0e-05 loss: 5.762e-02] jam: 5%|▌ | 1014/20000 [04:53<1:25:29, 3.70it/s, lr: 7.0e-05 loss: 8.110e-02] jam: 5%|▌ | 1015/20000 [04:53<1:45:36, 3.00it/s, lr: 7.0e-05 loss: 8.110e-02] jam: 5%|▌ | 1015/20000 [04:53<1:45:36, 3.00it/s, lr: 7.0e-05 loss: 1.853e-01] jam: 5%|▌ | 1016/20000 [04:53<1:12:58, 4.34it/s, lr: 7.0e-05 loss: 1.853e-01] jam: 5%|▌ | 1016/20000 [04:54<1:12:58, 4.34it/s, lr: 7.0e-05 loss: 1.730e-01] jam: 5%|▌ | 1017/20000 [04:54<1:16:20, 4.14it/s, lr: 7.0e-05 loss: 1.730e-01] jam: 5%|▌ | 1017/20000 [04:54<1:16:20, 4.14it/s, lr: 7.0e-05 loss: 2.021e-01] jam: 5%|▌ | 1018/20000 [04:54<1:10:30, 4.49it/s, lr: 7.0e-05 loss: 2.021e-01] jam: 5%|▌ | 1018/20000 [04:54<1:10:30, 4.49it/s, lr: 7.0e-05 loss: 2.216e-01] jam: 5%|▌ | 1019/20000 [04:54<1:38:32, 3.21it/s, lr: 7.0e-05 loss: 2.216e-01] jam: 5%|▌ | 1019/20000 [04:54<1:38:32, 3.21it/s, lr: 7.0e-05 loss: 2.896e-01] jam: 5%|▌ | 1020/20000 [04:54<1:13:04, 4.33it/s, lr: 7.0e-05 loss: 2.896e-01] jam: 5%|▌ | 1020/20000 [04:55<1:13:04, 4.33it/s, lr: 7.0e-05 loss: 5.882e-02] jam: 5%|▌ | 1021/20000 [04:55<1:36:46, 3.27it/s, lr: 7.0e-05 loss: 5.882e-02] jam: 5%|▌ | 1021/20000 [04:55<1:36:46, 3.27it/s, lr: 7.0e-05 loss: 5.452e-02] jam: 5%|▌ | 1022/20000 [04:55<1:51:06, 2.85it/s, lr: 7.0e-05 loss: 5.452e-02] jam: 5%|▌ | 1022/20000 [04:55<1:51:06, 2.85it/s, lr: 7.0e-05 loss: 1.143e-01] jam: 5%|▌ | 1023/20000 [04:55<1:13:17, 4.32it/s, lr: 7.0e-05 loss: 1.143e-01] jam: 5%|▌ | 1023/20000 [04:56<1:13:17, 4.32it/s, lr: 7.0e-05 loss: 1.034e-01] jam: 5%|▌ | 1024/20000 [04:56<1:16:56, 4.11it/s, lr: 7.0e-05 loss: 1.034e-01] jam: 5%|▌ | 1024/20000 [04:56<1:16:56, 4.11it/s, lr: 7.0e-05 loss: 9.272e-02] jam: 5%|▌ | 1025/20000 [04:56<1:45:18, 3.00it/s, lr: 7.0e-05 loss: 9.272e-02] jam: 5%|▌ | 1025/20000 [04:56<1:45:18, 3.00it/s, lr: 7.0e-05 loss: 2.575e-01] | |
| Saving at step 1025 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001025.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▌ | 1026/20000 [04:57<1:45:17, 3.00it/s, lr: 7.0e-05 loss: 1.012e-01] jam: 5%|▌ | 1027/20000 [04:57<1:41:37, 3.11it/s, lr: 7.0e-05 loss: 1.012e-01] jam: 5%|▌ | 1027/20000 [04:57<1:41:37, 3.11it/s, lr: 7.0e-05 loss: 1.401e-01] jam: 5%|▌ | 1028/20000 [04:57<1:24:15, 3.75it/s, lr: 7.0e-05 loss: 1.401e-01] jam: 5%|▌ | 1028/20000 [04:57<1:24:15, 3.75it/s, lr: 7.0e-05 loss: 1.359e-01] jam: 5%|▌ | 1029/20000 [04:57<1:15:30, 4.19it/s, lr: 7.0e-05 loss: 1.359e-01] jam: 5%|▌ | 1029/20000 [04:58<1:15:30, 4.19it/s, lr: 7.0e-05 loss: 1.711e-01] jam: 5%|▌ | 1030/20000 [04:58<1:39:30, 3.18it/s, lr: 7.0e-05 loss: 1.711e-01] jam: 5%|▌ | 1030/20000 [04:58<1:39:30, 3.18it/s, lr: 7.0e-05 loss: 9.793e-02] jam: 5%|▌ | 1031/20000 [04:58<1:19:14, 3.99it/s, lr: 7.0e-05 loss: 9.793e-02] jam: 5%|▌ | 1031/20000 [04:58<1:19:14, 3.99it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 5%|▌ | 1032/20000 [04:58<1:44:04, 3.04it/s, lr: 7.0e-05 loss: 1.703e-01] jam: 5%|▌ | 1032/20000 [04:58<1:44:04, 3.04it/s, lr: 7.0e-05 loss: 2.361e-01] jam: 5%|▌ | 1033/20000 [04:58<1:09:58, 4.52it/s, lr: 7.0e-05 loss: 2.361e-01] jam: 5%|▌ | 1033/20000 [04:59<1:09:58, 4.52it/s, lr: 7.0e-05 loss: 9.657e-02] jam: 5%|▌ | 1034/20000 [04:59<1:35:51, 3.30it/s, lr: 7.0e-05 loss: 9.657e-02] jam: 5%|▌ | 1034/20000 [04:59<1:35:51, 3.30it/s, lr: 7.0e-05 loss: 1.433e-01] jam: 5%|▌ | 1035/20000 [04:59<1:52:03, 2.82it/s, lr: 7.0e-05 loss: 1.433e-01] jam: 5%|▌ | 1035/20000 [04:59<1:52:03, 2.82it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 5%|▌ | 1036/20000 [04:59<1:48:48, 2.90it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 5%|▌ | 1036/20000 [05:00<1:48:48, 2.90it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 5%|▌ | 1037/20000 [05:00<1:52:21, 2.81it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 5%|▌ | 1037/20000 [05:00<1:52:21, 2.81it/s, lr: 7.0e-05 loss: 1.198e-01] jam: 5%|▌ | 1038/20000 [05:00<1:14:06, 4.26it/s, lr: 7.0e-05 loss: 1.198e-01] jam: 5%|▌ | 1038/20000 [05:00<1:14:06, 4.26it/s, lr: 7.0e-05 loss: 1.107e-01] jam: 5%|▌ | 1039/20000 [05:00<1:06:12, 4.77it/s, lr: 7.0e-05 loss: 1.107e-01] jam: 5%|▌ | 1039/20000 [05:00<1:06:12, 4.77it/s, lr: 7.0e-05 loss: 1.038e-01] jam: 5%|▌ | 1040/20000 [05:00<1:04:08, 4.93it/s, lr: 7.0e-05 loss: 1.038e-01] jam: 5%|▌ | 1040/20000 [05:01<1:04:08, 4.93it/s, lr: 7.0e-05 loss: 3.255e-02] jam: 5%|▌ | 1041/20000 [05:01<1:34:42, 3.34it/s, lr: 7.0e-05 loss: 3.255e-02] jam: 5%|▌ | 1041/20000 [05:01<1:34:42, 3.34it/s, lr: 7.0e-05 loss: 1.203e-01] jam: 5%|▌ | 1042/20000 [05:01<1:43:22, 3.06it/s, lr: 7.0e-05 loss: 1.203e-01] jam: 5%|▌ | 1042/20000 [05:01<1:43:22, 3.06it/s, lr: 7.0e-05 loss: 2.010e-01] jam: 5%|▌ | 1043/20000 [05:01<1:50:59, 2.85it/s, lr: 7.0e-05 loss: 2.010e-01] jam: 5%|▌ | 1043/20000 [05:02<1:50:59, 2.85it/s, lr: 7.0e-05 loss: 1.915e-01] jam: 5%|▌ | 1044/20000 [05:02<1:24:45, 3.73it/s, lr: 7.0e-05 loss: 1.915e-01] jam: 5%|▌ | 1044/20000 [05:02<1:24:45, 3.73it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 5%|▌ | 1045/20000 [05:02<1:25:06, 3.71it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 5%|▌ | 1045/20000 [05:02<1:25:06, 3.71it/s, lr: 7.0e-05 loss: 1.904e-01] jam: 5%|▌ | 1046/20000 [05:02<1:12:05, 4.38it/s, lr: 7.0e-05 loss: 1.904e-01] jam: 5%|▌ | 1046/20000 [05:02<1:12:05, 4.38it/s, lr: 7.0e-05 loss: 8.216e-02] jam: 5%|▌ | 1047/20000 [05:02<1:27:16, 3.62it/s, lr: 7.0e-05 loss: 8.216e-02] jam: 5%|▌ | 1047/20000 [05:03<1:27:16, 3.62it/s, lr: 7.0e-05 loss: 1.222e-01] jam: 5%|▌ | 1048/20000 [05:03<1:47:26, 2.94it/s, lr: 7.0e-05 loss: 1.222e-01] jam: 5%|▌ | 1048/20000 [05:03<1:47:26, 2.94it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 5%|▌ | 1049/20000 [05:03<1:45:57, 2.98it/s, lr: 7.0e-05 loss: 1.114e-01] jam: 5%|▌ | 1049/20000 [05:03<1:45:57, 2.98it/s, lr: 7.0e-05 loss: 1.385e-01] jam: 5%|▌ | 1050/20000 [05:03<1:52:25, 2.81it/s, lr: 7.0e-05 loss: 1.385e-01] jam: 5%|▌ | 1050/20000 [05:04<1:52:25, 2.81it/s, lr: 7.0e-05 loss: 1.341e-01] | |
| Saving at step 1050 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001050.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▌ | 1051/20000 [05:04<1:52:25, 2.81it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 5%|▌ | 1052/20000 [05:04<1:44:04, 3.03it/s, lr: 7.0e-05 loss: 1.607e-01] jam: 5%|▌ | 1052/20000 [05:04<1:44:04, 3.03it/s, lr: 7.0e-05 loss: 1.094e-01] jam: 5%|▌ | 1053/20000 [05:04<1:15:45, 4.17it/s, lr: 7.0e-05 loss: 1.094e-01] jam: 5%|▌ | 1053/20000 [05:05<1:15:45, 4.17it/s, lr: 7.0e-05 loss: 2.424e-01] jam: 5%|▌ | 1054/20000 [05:05<1:05:04, 4.85it/s, lr: 7.0e-05 loss: 2.424e-01] jam: 5%|▌ | 1054/20000 [05:05<1:05:04, 4.85it/s, lr: 7.0e-05 loss: 2.001e-01] jam: 5%|▌ | 1055/20000 [05:05<1:03:34, 4.97it/s, lr: 7.0e-05 loss: 2.001e-01] jam: 5%|▌ | 1055/20000 [05:05<1:03:34, 4.97it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 5%|▌ | 1056/20000 [05:05<1:10:02, 4.51it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 5%|▌ | 1056/20000 [05:05<1:10:02, 4.51it/s, lr: 7.0e-05 loss: 2.257e-01] jam: 5%|▌ | 1057/20000 [05:05<1:05:14, 4.84it/s, lr: 7.0e-05 loss: 2.257e-01] jam: 5%|▌ | 1057/20000 [05:05<1:05:14, 4.84it/s, lr: 7.0e-05 loss: 2.206e-01] jam: 5%|▌ | 1058/20000 [05:05<1:04:20, 4.91it/s, lr: 7.0e-05 loss: 2.206e-01] jam: 5%|▌ | 1058/20000 [05:06<1:04:20, 4.91it/s, lr: 7.0e-05 loss: 1.027e-01] jam: 5%|▌ | 1059/20000 [05:06<1:33:56, 3.36it/s, lr: 7.0e-05 loss: 1.027e-01] jam: 5%|▌ | 1059/20000 [05:06<1:33:56, 3.36it/s, lr: 7.0e-05 loss: 1.142e-01] jam: 5%|▌ | 1060/20000 [05:06<1:49:09, 2.89it/s, lr: 7.0e-05 loss: 1.142e-01] jam: 5%|▌ | 1060/20000 [05:07<1:49:09, 2.89it/s, lr: 7.0e-05 loss: 2.827e-01] jam: 5%|▌ | 1061/20000 [05:07<1:46:23, 2.97it/s, lr: 7.0e-05 loss: 2.827e-01] jam: 5%|▌ | 1061/20000 [05:07<1:46:23, 2.97it/s, lr: 7.0e-05 loss: 9.320e-02] jam: 5%|▌ | 1062/20000 [05:07<1:19:43, 3.96it/s, lr: 7.0e-05 loss: 9.320e-02] jam: 5%|▌ | 1062/20000 [05:07<1:19:43, 3.96it/s, lr: 7.0e-05 loss: 1.738e-01] jam: 5%|▌ | 1063/20000 [05:07<1:09:17, 4.55it/s, lr: 7.0e-05 loss: 1.738e-01] jam: 5%|▌ | 1063/20000 [05:07<1:09:17, 4.55it/s, lr: 7.0e-05 loss: 9.040e-02] jam: 5%|▌ | 1064/20000 [05:07<1:17:11, 4.09it/s, lr: 7.0e-05 loss: 9.040e-02] jam: 5%|▌ | 1064/20000 [05:07<1:17:11, 4.09it/s, lr: 7.0e-05 loss: 7.298e-02] jam: 5%|▌ | 1065/20000 [05:07<1:12:16, 4.37it/s, lr: 7.0e-05 loss: 7.298e-02] jam: 5%|▌ | 1065/20000 [05:08<1:12:16, 4.37it/s, lr: 7.0e-05 loss: 1.428e-01] jam: 5%|▌ | 1066/20000 [05:08<1:11:46, 4.40it/s, lr: 7.0e-05 loss: 1.428e-01] jam: 5%|▌ | 1066/20000 [05:08<1:11:46, 4.40it/s, lr: 7.0e-05 loss: 2.300e-01] jam: 5%|▌ | 1067/20000 [05:08<1:06:13, 4.76it/s, lr: 7.0e-05 loss: 2.300e-01] jam: 5%|▌ | 1067/20000 [05:08<1:06:13, 4.76it/s, lr: 7.0e-05 loss: 1.530e-01] jam: 5%|▌ | 1068/20000 [05:08<1:35:52, 3.29it/s, lr: 7.0e-05 loss: 1.530e-01] jam: 5%|▌ | 1068/20000 [05:09<1:35:52, 3.29it/s, lr: 7.0e-05 loss: 1.926e-01] jam: 5%|▌ | 1069/20000 [05:09<1:51:40, 2.83it/s, lr: 7.0e-05 loss: 1.926e-01] jam: 5%|▌ | 1069/20000 [05:09<1:51:40, 2.83it/s, lr: 7.0e-05 loss: 1.180e-01] jam: 5%|▌ | 1070/20000 [05:09<1:14:09, 4.25it/s, lr: 7.0e-05 loss: 1.180e-01] jam: 5%|▌ | 1070/20000 [05:09<1:14:09, 4.25it/s, lr: 7.0e-05 loss: 2.755e-01] jam: 5%|▌ | 1071/20000 [05:09<1:37:48, 3.23it/s, lr: 7.0e-05 loss: 2.755e-01] jam: 5%|▌ | 1071/20000 [05:09<1:37:48, 3.23it/s, lr: 7.0e-05 loss: 1.665e-01] jam: 5%|▌ | 1072/20000 [05:09<1:18:51, 4.00it/s, lr: 7.0e-05 loss: 1.665e-01] jam: 5%|▌ | 1072/20000 [05:10<1:18:51, 4.00it/s, lr: 7.0e-05 loss: 9.579e-02] jam: 5%|▌ | 1073/20000 [05:10<1:46:31, 2.96it/s, lr: 7.0e-05 loss: 9.579e-02] jam: 5%|▌ | 1073/20000 [05:10<1:46:31, 2.96it/s, lr: 7.0e-05 loss: 5.199e-02] jam: 5%|▌ | 1074/20000 [05:10<1:53:19, 2.78it/s, lr: 7.0e-05 loss: 5.199e-02] jam: 5%|▌ | 1074/20000 [05:10<1:53:19, 2.78it/s, lr: 7.0e-05 loss: 2.028e-01] jam: 5%|▌ | 1075/20000 [05:10<1:22:38, 3.82it/s, lr: 7.0e-05 loss: 2.028e-01] jam: 5%|▌ | 1075/20000 [05:11<1:22:38, 3.82it/s, lr: 7.0e-05 loss: 1.339e-01] | |
| Saving at step 1075 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001075.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 5%|▌ | 1076/20000 [05:11<1:22:38, 3.82it/s, lr: 7.0e-05 loss: 2.643e-01] jam: 5%|▌ | 1077/20000 [05:11<1:21:44, 3.86it/s, lr: 7.0e-05 loss: 2.643e-01] jam: 5%|▌ | 1077/20000 [05:11<1:21:44, 3.86it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 5%|▌ | 1078/20000 [05:11<1:09:00, 4.57it/s, lr: 7.0e-05 loss: 1.091e-01] jam: 5%|▌ | 1078/20000 [05:12<1:09:00, 4.57it/s, lr: 7.0e-05 loss: 2.379e-01] jam: 5%|▌ | 1079/20000 [05:12<1:36:50, 3.26it/s, lr: 7.0e-05 loss: 2.379e-01] jam: 5%|▌ | 1079/20000 [05:12<1:36:50, 3.26it/s, lr: 7.0e-05 loss: 6.156e-02] jam: 5%|▌ | 1080/20000 [05:12<1:49:32, 2.88it/s, lr: 7.0e-05 loss: 6.156e-02] jam: 5%|▌ | 1080/20000 [05:12<1:49:32, 2.88it/s, lr: 7.0e-05 loss: 9.387e-02] jam: 5%|▌ | 1081/20000 [05:12<1:48:20, 2.91it/s, lr: 7.0e-05 loss: 9.387e-02] jam: 5%|▌ | 1081/20000 [05:13<1:48:20, 2.91it/s, lr: 7.0e-05 loss: 1.061e-01] jam: 5%|▌ | 1082/20000 [05:13<1:21:21, 3.88it/s, lr: 7.0e-05 loss: 1.061e-01] jam: 5%|▌ | 1082/20000 [05:13<1:21:21, 3.88it/s, lr: 7.0e-05 loss: 9.142e-02] jam: 5%|▌ | 1083/20000 [05:13<1:16:39, 4.11it/s, lr: 7.0e-05 loss: 9.142e-02] jam: 5%|▌ | 1083/20000 [05:13<1:16:39, 4.11it/s, lr: 7.0e-05 loss: 2.154e-01] jam: 5%|▌ | 1084/20000 [05:13<1:03:57, 4.93it/s, lr: 7.0e-05 loss: 2.154e-01] jam: 5%|▌ | 1084/20000 [05:13<1:03:57, 4.93it/s, lr: 7.0e-05 loss: 9.095e-02] jam: 5%|▌ | 1085/20000 [05:13<1:34:23, 3.34it/s, lr: 7.0e-05 loss: 9.095e-02] jam: 5%|▌ | 1085/20000 [05:14<1:34:23, 3.34it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 5%|▌ | 1086/20000 [05:14<1:20:19, 3.92it/s, lr: 7.0e-05 loss: 1.690e-01] jam: 5%|▌ | 1086/20000 [05:14<1:20:19, 3.92it/s, lr: 7.0e-05 loss: 1.350e-01] jam: 5%|▌ | 1087/20000 [05:14<1:41:55, 3.09it/s, lr: 7.0e-05 loss: 1.350e-01] jam: 5%|▌ | 1087/20000 [05:14<1:41:55, 3.09it/s, lr: 7.0e-05 loss: 1.418e-01] jam: 5%|▌ | 1088/20000 [05:14<1:09:00, 4.57it/s, lr: 7.0e-05 loss: 1.418e-01] jam: 5%|▌ | 1088/20000 [05:14<1:09:00, 4.57it/s, lr: 7.0e-05 loss: 1.189e-01] jam: 5%|▌ | 1089/20000 [05:14<1:06:55, 4.71it/s, lr: 7.0e-05 loss: 1.189e-01] jam: 5%|▌ | 1089/20000 [05:15<1:06:55, 4.71it/s, lr: 7.0e-05 loss: 1.676e-01] jam: 5%|▌ | 1090/20000 [05:15<1:35:00, 3.32it/s, lr: 7.0e-05 loss: 1.676e-01] jam: 5%|▌ | 1090/20000 [05:15<1:35:00, 3.32it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 5%|▌ | 1091/20000 [05:15<1:48:30, 2.90it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 5%|▌ | 1091/20000 [05:15<1:48:30, 2.90it/s, lr: 7.0e-05 loss: 1.310e-01] jam: 5%|▌ | 1092/20000 [05:15<1:12:35, 4.34it/s, lr: 7.0e-05 loss: 1.310e-01] jam: 5%|▌ | 1092/20000 [05:15<1:12:35, 4.34it/s, lr: 7.0e-05 loss: 2.635e-01] jam: 5%|▌ | 1093/20000 [05:15<1:06:05, 4.77it/s, lr: 7.0e-05 loss: 2.635e-01] jam: 5%|▌ | 1093/20000 [05:16<1:06:05, 4.77it/s, lr: 7.0e-05 loss: 1.609e-01] jam: 5%|▌ | 1094/20000 [05:16<1:00:16, 5.23it/s, lr: 7.0e-05 loss: 1.609e-01] jam: 5%|▌ | 1094/20000 [05:16<1:00:16, 5.23it/s, lr: 7.0e-05 loss: 1.262e-01] jam: 5%|▌ | 1095/20000 [05:16<1:05:30, 4.81it/s, lr: 7.0e-05 loss: 1.262e-01] jam: 5%|▌ | 1095/20000 [05:16<1:05:30, 4.81it/s, lr: 7.0e-05 loss: 1.072e-01] jam: 5%|▌ | 1096/20000 [05:16<1:04:29, 4.89it/s, lr: 7.0e-05 loss: 1.072e-01] jam: 5%|▌ | 1096/20000 [05:16<1:04:29, 4.89it/s, lr: 7.0e-05 loss: 1.132e-01] jam: 5%|▌ | 1097/20000 [05:16<1:03:18, 4.98it/s, lr: 7.0e-05 loss: 1.132e-01] jam: 5%|▌ | 1097/20000 [05:16<1:03:18, 4.98it/s, lr: 7.0e-05 loss: 2.214e-01] jam: 5%|▌ | 1098/20000 [05:16<1:04:20, 4.90it/s, lr: 7.0e-05 loss: 2.214e-01] jam: 5%|▌ | 1098/20000 [05:17<1:04:20, 4.90it/s, lr: 7.0e-05 loss: 3.680e-02] jam: 5%|▌ | 1099/20000 [05:17<1:33:44, 3.36it/s, lr: 7.0e-05 loss: 3.680e-02] jam: 5%|▌ | 1099/20000 [05:17<1:33:44, 3.36it/s, lr: 7.0e-05 loss: 5.888e-02] jam: 6%|▌ | 1100/20000 [05:17<1:19:20, 3.97it/s, lr: 7.0e-05 loss: 5.888e-02] jam: 6%|▌ | 1100/20000 [05:17<1:19:20, 3.97it/s, lr: 7.0e-05 loss: 5.428e-02] | |
| Saving at step 1100 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001100.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▌ | 1101/20000 [05:18<1:19:20, 3.97it/s, lr: 7.0e-05 loss: 1.285e-01] jam: 6%|▌ | 1102/20000 [05:18<1:36:13, 3.27it/s, lr: 7.0e-05 loss: 1.285e-01] jam: 6%|▌ | 1102/20000 [05:18<1:36:13, 3.27it/s, lr: 7.0e-05 loss: 1.796e-01] jam: 6%|▌ | 1103/20000 [05:18<1:48:57, 2.89it/s, lr: 7.0e-05 loss: 1.796e-01] jam: 6%|▌ | 1103/20000 [05:19<1:48:57, 2.89it/s, lr: 7.0e-05 loss: 1.215e-01] jam: 6%|▌ | 1104/20000 [05:19<1:47:54, 2.92it/s, lr: 7.0e-05 loss: 1.215e-01] jam: 6%|▌ | 1104/20000 [05:19<1:47:54, 2.92it/s, lr: 7.0e-05 loss: 2.135e-01] jam: 6%|▌ | 1105/20000 [05:19<1:20:55, 3.89it/s, lr: 7.0e-05 loss: 2.135e-01] jam: 6%|▌ | 1105/20000 [05:19<1:20:55, 3.89it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 6%|▌ | 1106/20000 [05:19<1:13:41, 4.27it/s, lr: 7.0e-05 loss: 2.065e-01] jam: 6%|▌ | 1106/20000 [05:19<1:13:41, 4.27it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 6%|▌ | 1107/20000 [05:19<1:06:48, 4.71it/s, lr: 7.0e-05 loss: 1.138e-01] jam: 6%|▌ | 1107/20000 [05:19<1:06:48, 4.71it/s, lr: 7.0e-05 loss: 1.197e-01] jam: 6%|▌ | 1108/20000 [05:19<1:04:45, 4.86it/s, lr: 7.0e-05 loss: 1.197e-01] jam: 6%|▌ | 1108/20000 [05:20<1:04:45, 4.86it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 6%|▌ | 1109/20000 [05:20<1:00:50, 5.17it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 6%|▌ | 1109/20000 [05:20<1:00:50, 5.17it/s, lr: 7.0e-05 loss: 2.168e-01] jam: 6%|▌ | 1110/20000 [05:20<1:02:03, 5.07it/s, lr: 7.0e-05 loss: 2.168e-01] jam: 6%|▌ | 1110/20000 [05:20<1:02:03, 5.07it/s, lr: 7.0e-05 loss: 8.662e-02] jam: 6%|▌ | 1111/20000 [05:20<1:07:25, 4.67it/s, lr: 7.0e-05 loss: 8.662e-02] jam: 6%|▌ | 1111/20000 [05:20<1:07:25, 4.67it/s, lr: 7.0e-05 loss: 1.623e-01] jam: 6%|▌ | 1112/20000 [05:20<1:33:30, 3.37it/s, lr: 7.0e-05 loss: 1.623e-01] jam: 6%|▌ | 1112/20000 [05:21<1:33:30, 3.37it/s, lr: 7.0e-05 loss: 3.439e-02] jam: 6%|▌ | 1113/20000 [05:21<1:50:16, 2.85it/s, lr: 7.0e-05 loss: 3.439e-02] jam: 6%|▌ | 1113/20000 [05:21<1:50:16, 2.85it/s, lr: 7.0e-05 loss: 3.226e-01] jam: 6%|▌ | 1114/20000 [05:21<1:13:27, 4.29it/s, lr: 7.0e-05 loss: 3.226e-01] jam: 6%|▌ | 1114/20000 [05:21<1:13:27, 4.29it/s, lr: 7.0e-05 loss: 8.590e-02] jam: 6%|▌ | 1115/20000 [05:21<1:05:50, 4.78it/s, lr: 7.0e-05 loss: 8.590e-02] jam: 6%|▌ | 1115/20000 [05:21<1:05:50, 4.78it/s, lr: 7.0e-05 loss: 5.988e-02] jam: 6%|▌ | 1116/20000 [05:21<1:34:33, 3.33it/s, lr: 7.0e-05 loss: 5.988e-02] jam: 6%|▌ | 1116/20000 [05:22<1:34:33, 3.33it/s, lr: 7.0e-05 loss: 2.668e-01] jam: 6%|▌ | 1117/20000 [05:22<1:17:48, 4.05it/s, lr: 7.0e-05 loss: 2.668e-01] jam: 6%|▌ | 1117/20000 [05:22<1:17:48, 4.05it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 6%|▌ | 1118/20000 [05:22<1:43:45, 3.03it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 6%|▌ | 1118/20000 [05:22<1:43:45, 3.03it/s, lr: 7.0e-05 loss: 7.005e-02] jam: 6%|▌ | 1119/20000 [05:22<1:45:30, 2.98it/s, lr: 7.0e-05 loss: 7.005e-02] jam: 6%|▌ | 1119/20000 [05:23<1:45:30, 2.98it/s, lr: 7.0e-05 loss: 1.201e-01] jam: 6%|▌ | 1120/20000 [05:23<1:53:26, 2.77it/s, lr: 7.0e-05 loss: 1.201e-01] jam: 6%|▌ | 1120/20000 [05:23<1:53:26, 2.77it/s, lr: 7.0e-05 loss: 8.544e-02] jam: 6%|▌ | 1121/20000 [05:23<1:22:25, 3.82it/s, lr: 7.0e-05 loss: 8.544e-02] jam: 6%|▌ | 1121/20000 [05:23<1:22:25, 3.82it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 6%|▌ | 1122/20000 [05:23<1:45:33, 2.98it/s, lr: 7.0e-05 loss: 1.266e-01] jam: 6%|▌ | 1122/20000 [05:24<1:45:33, 2.98it/s, lr: 7.0e-05 loss: 1.416e-01] jam: 6%|▌ | 1123/20000 [05:24<1:46:39, 2.95it/s, lr: 7.0e-05 loss: 1.416e-01] jam: 6%|▌ | 1123/20000 [05:24<1:46:39, 2.95it/s, lr: 7.0e-05 loss: 9.379e-02] jam: 6%|▌ | 1124/20000 [05:24<1:53:39, 2.77it/s, lr: 7.0e-05 loss: 9.379e-02] jam: 6%|▌ | 1124/20000 [05:24<1:53:39, 2.77it/s, lr: 7.0e-05 loss: 1.754e-01] jam: 6%|▌ | 1125/20000 [05:24<1:48:56, 2.89it/s, lr: 7.0e-05 loss: 1.754e-01] jam: 6%|▌ | 1125/20000 [05:25<1:48:56, 2.89it/s, lr: 7.0e-05 loss: 1.321e-01] | |
| Saving at step 1125 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001125.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▌ | 1126/20000 [05:25<1:48:56, 2.89it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 6%|▌ | 1127/20000 [05:25<1:23:28, 3.77it/s, lr: 7.0e-05 loss: 9.697e-02] jam: 6%|▌ | 1127/20000 [05:25<1:23:28, 3.77it/s, lr: 7.0e-05 loss: 2.210e-01] jam: 6%|▌ | 1128/20000 [05:25<1:10:21, 4.47it/s, lr: 7.0e-05 loss: 2.210e-01] jam: 6%|▌ | 1128/20000 [05:26<1:10:21, 4.47it/s, lr: 7.0e-05 loss: 1.173e-01] jam: 6%|▌ | 1129/20000 [05:26<1:34:28, 3.33it/s, lr: 7.0e-05 loss: 1.173e-01] jam: 6%|▌ | 1129/20000 [05:26<1:34:28, 3.33it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 6%|▌ | 1130/20000 [05:26<1:49:51, 2.86it/s, lr: 7.0e-05 loss: 1.133e-01] jam: 6%|▌ | 1130/20000 [05:26<1:49:51, 2.86it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 6%|▌ | 1131/20000 [05:26<1:22:47, 3.80it/s, lr: 7.0e-05 loss: 1.158e-01] jam: 6%|▌ | 1131/20000 [05:26<1:22:47, 3.80it/s, lr: 7.0e-05 loss: 1.962e-01] jam: 6%|▌ | 1132/20000 [05:26<1:13:08, 4.30it/s, lr: 7.0e-05 loss: 1.962e-01] jam: 6%|▌ | 1132/20000 [05:27<1:13:08, 4.30it/s, lr: 7.0e-05 loss: 2.854e-01] jam: 6%|▌ | 1133/20000 [05:27<1:05:41, 4.79it/s, lr: 7.0e-05 loss: 2.854e-01] jam: 6%|▌ | 1133/20000 [05:27<1:05:41, 4.79it/s, lr: 7.0e-05 loss: 6.868e-02] jam: 6%|▌ | 1134/20000 [05:27<1:06:47, 4.71it/s, lr: 7.0e-05 loss: 6.868e-02] jam: 6%|▌ | 1134/20000 [05:27<1:06:47, 4.71it/s, lr: 7.0e-05 loss: 2.631e-01] jam: 6%|▌ | 1135/20000 [05:27<1:43:10, 3.05it/s, lr: 7.0e-05 loss: 2.631e-01] jam: 6%|▌ | 1135/20000 [05:27<1:43:10, 3.05it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 6%|▌ | 1136/20000 [05:27<1:17:43, 4.04it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 6%|▌ | 1136/20000 [05:28<1:17:43, 4.04it/s, lr: 7.0e-05 loss: 1.227e-01] jam: 6%|▌ | 1137/20000 [05:28<1:44:38, 3.00it/s, lr: 7.0e-05 loss: 1.227e-01] jam: 6%|▌ | 1137/20000 [05:28<1:44:38, 3.00it/s, lr: 7.0e-05 loss: 1.831e-01] jam: 6%|▌ | 1138/20000 [05:28<1:17:25, 4.06it/s, lr: 7.0e-05 loss: 1.831e-01] jam: 6%|▌ | 1138/20000 [05:28<1:17:25, 4.06it/s, lr: 7.0e-05 loss: 1.077e-01] jam: 6%|▌ | 1139/20000 [05:28<1:12:11, 4.35it/s, lr: 7.0e-05 loss: 1.077e-01] jam: 6%|▌ | 1139/20000 [05:28<1:12:11, 4.35it/s, lr: 7.0e-05 loss: 1.918e-01] jam: 6%|▌ | 1140/20000 [05:28<1:09:15, 4.54it/s, lr: 7.0e-05 loss: 1.918e-01] jam: 6%|▌ | 1140/20000 [05:29<1:09:15, 4.54it/s, lr: 7.0e-05 loss: 1.913e-01] jam: 6%|▌ | 1141/20000 [05:29<1:08:58, 4.56it/s, lr: 7.0e-05 loss: 1.913e-01] jam: 6%|▌ | 1141/20000 [05:29<1:08:58, 4.56it/s, lr: 7.0e-05 loss: 2.503e-01] jam: 6%|▌ | 1142/20000 [05:29<1:03:50, 4.92it/s, lr: 7.0e-05 loss: 2.503e-01] jam: 6%|▌ | 1142/20000 [05:29<1:03:50, 4.92it/s, lr: 7.0e-05 loss: 2.120e-01] jam: 6%|▌ | 1143/20000 [05:29<1:02:56, 4.99it/s, lr: 7.0e-05 loss: 2.120e-01] jam: 6%|▌ | 1143/20000 [05:29<1:02:56, 4.99it/s, lr: 7.0e-05 loss: 1.128e-01] jam: 6%|▌ | 1144/20000 [05:29<1:34:16, 3.33it/s, lr: 7.0e-05 loss: 1.128e-01] jam: 6%|▌ | 1144/20000 [05:30<1:34:16, 3.33it/s, lr: 7.0e-05 loss: 1.039e-01] jam: 6%|▌ | 1145/20000 [05:30<1:51:35, 2.82it/s, lr: 7.0e-05 loss: 1.039e-01] jam: 6%|▌ | 1145/20000 [05:30<1:51:35, 2.82it/s, lr: 7.0e-05 loss: 1.299e-01] jam: 6%|▌ | 1146/20000 [05:30<1:22:45, 3.80it/s, lr: 7.0e-05 loss: 1.299e-01] jam: 6%|▌ | 1146/20000 [05:30<1:22:45, 3.80it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 6%|▌ | 1147/20000 [05:30<1:13:56, 4.25it/s, lr: 7.0e-05 loss: 1.115e-01] jam: 6%|▌ | 1147/20000 [05:31<1:13:56, 4.25it/s, lr: 7.0e-05 loss: 7.552e-02] jam: 6%|▌ | 1148/20000 [05:31<1:47:43, 2.92it/s, lr: 7.0e-05 loss: 7.552e-02] jam: 6%|▌ | 1148/20000 [05:31<1:47:43, 2.92it/s, lr: 7.0e-05 loss: 3.255e-01] jam: 6%|▌ | 1149/20000 [05:31<1:29:28, 3.51it/s, lr: 7.0e-05 loss: 3.255e-01] jam: 6%|▌ | 1149/20000 [05:31<1:29:28, 3.51it/s, lr: 7.0e-05 loss: 2.288e-01] jam: 6%|▌ | 1150/20000 [05:31<1:41:44, 3.09it/s, lr: 7.0e-05 loss: 2.288e-01] jam: 6%|▌ | 1150/20000 [05:31<1:41:44, 3.09it/s, lr: 7.0e-05 loss: 2.168e-01] | |
| Saving at step 1150 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001150.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▌ | 1151/20000 [05:32<1:41:43, 3.09it/s, lr: 7.0e-05 loss: 1.794e-01] jam: 6%|▌ | 1152/20000 [05:32<1:38:59, 3.17it/s, lr: 7.0e-05 loss: 1.794e-01] jam: 6%|▌ | 1152/20000 [05:32<1:38:59, 3.17it/s, lr: 7.0e-05 loss: 2.612e-01] jam: 6%|▌ | 1153/20000 [05:32<1:21:07, 3.87it/s, lr: 7.0e-05 loss: 2.612e-01] jam: 6%|▌ | 1153/20000 [05:33<1:21:07, 3.87it/s, lr: 7.0e-05 loss: 1.672e-01] jam: 6%|▌ | 1154/20000 [05:33<1:13:10, 4.29it/s, lr: 7.0e-05 loss: 1.672e-01] jam: 6%|▌ | 1154/20000 [05:33<1:13:10, 4.29it/s, lr: 7.0e-05 loss: 1.196e-01] jam: 6%|▌ | 1155/20000 [05:33<1:38:15, 3.20it/s, lr: 7.0e-05 loss: 1.196e-01] jam: 6%|▌ | 1155/20000 [05:33<1:38:15, 3.20it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 6%|▌ | 1156/20000 [05:33<1:19:51, 3.93it/s, lr: 7.0e-05 loss: 1.882e-01] jam: 6%|▌ | 1156/20000 [05:33<1:19:51, 3.93it/s, lr: 7.0e-05 loss: 2.718e-01] jam: 6%|▌ | 1157/20000 [05:33<1:12:46, 4.31it/s, lr: 7.0e-05 loss: 2.718e-01] jam: 6%|▌ | 1157/20000 [05:34<1:12:46, 4.31it/s, lr: 7.0e-05 loss: 7.551e-02] jam: 6%|▌ | 1158/20000 [05:34<1:38:23, 3.19it/s, lr: 7.0e-05 loss: 7.551e-02] jam: 6%|▌ | 1158/20000 [05:34<1:38:23, 3.19it/s, lr: 7.0e-05 loss: 1.576e-01] jam: 6%|▌ | 1159/20000 [05:34<1:52:08, 2.80it/s, lr: 7.0e-05 loss: 1.576e-01] jam: 6%|▌ | 1159/20000 [05:34<1:52:08, 2.80it/s, lr: 7.0e-05 loss: 8.236e-02] jam: 6%|▌ | 1160/20000 [05:34<1:15:18, 4.17it/s, lr: 7.0e-05 loss: 8.236e-02] jam: 6%|▌ | 1160/20000 [05:35<1:15:18, 4.17it/s, lr: 7.0e-05 loss: 7.732e-02] jam: 6%|▌ | 1161/20000 [05:35<1:37:00, 3.24it/s, lr: 7.0e-05 loss: 7.732e-02] jam: 6%|▌ | 1161/20000 [05:35<1:37:00, 3.24it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 6%|▌ | 1162/20000 [05:35<1:15:30, 4.16it/s, lr: 7.0e-05 loss: 1.270e-01] jam: 6%|▌ | 1162/20000 [05:35<1:15:30, 4.16it/s, lr: 7.0e-05 loss: 1.343e-01] jam: 6%|▌ | 1163/20000 [05:35<1:08:19, 4.60it/s, lr: 7.0e-05 loss: 1.343e-01] jam: 6%|▌ | 1163/20000 [05:35<1:08:19, 4.60it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 6%|▌ | 1164/20000 [05:35<1:38:39, 3.18it/s, lr: 7.0e-05 loss: 1.827e-01] jam: 6%|▌ | 1164/20000 [05:36<1:38:39, 3.18it/s, lr: 7.0e-05 loss: 8.454e-02] jam: 6%|▌ | 1165/20000 [05:36<1:44:34, 3.00it/s, lr: 7.0e-05 loss: 8.454e-02] jam: 6%|▌ | 1165/20000 [05:36<1:44:34, 3.00it/s, lr: 7.0e-05 loss: 1.714e-01] jam: 6%|▌ | 1166/20000 [05:36<1:17:35, 4.05it/s, lr: 7.0e-05 loss: 1.714e-01] jam: 6%|▌ | 1166/20000 [05:36<1:17:35, 4.05it/s, lr: 7.0e-05 loss: 1.907e-01] jam: 6%|▌ | 1167/20000 [05:36<1:06:05, 4.75it/s, lr: 7.0e-05 loss: 1.907e-01] jam: 6%|▌ | 1167/20000 [05:36<1:06:05, 4.75it/s, lr: 7.0e-05 loss: 5.399e-02] jam: 6%|▌ | 1168/20000 [05:36<1:38:36, 3.18it/s, lr: 7.0e-05 loss: 5.399e-02] jam: 6%|▌ | 1168/20000 [05:37<1:38:36, 3.18it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 6%|▌ | 1169/20000 [05:37<1:43:56, 3.02it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 6%|▌ | 1169/20000 [05:37<1:43:56, 3.02it/s, lr: 7.0e-05 loss: 1.168e-01] jam: 6%|▌ | 1170/20000 [05:37<1:14:37, 4.21it/s, lr: 7.0e-05 loss: 1.168e-01] jam: 6%|▌ | 1170/20000 [05:37<1:14:37, 4.21it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 6%|▌ | 1171/20000 [05:37<1:08:27, 4.58it/s, lr: 7.0e-05 loss: 1.450e-01] jam: 6%|▌ | 1171/20000 [05:37<1:08:27, 4.58it/s, lr: 7.0e-05 loss: 1.096e-01] jam: 6%|▌ | 1172/20000 [05:37<1:14:22, 4.22it/s, lr: 7.0e-05 loss: 1.096e-01] jam: 6%|▌ | 1172/20000 [05:38<1:14:22, 4.22it/s, lr: 7.0e-05 loss: 6.589e-02] jam: 6%|▌ | 1173/20000 [05:38<1:08:41, 4.57it/s, lr: 7.0e-05 loss: 6.589e-02] jam: 6%|▌ | 1173/20000 [05:38<1:08:41, 4.57it/s, lr: 7.0e-05 loss: 9.177e-02] jam: 6%|▌ | 1174/20000 [05:38<58:42, 5.34it/s, lr: 7.0e-05 loss: 9.177e-02] jam: 6%|▌ | 1174/20000 [05:38<58:42, 5.34it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 6%|▌ | 1175/20000 [05:38<1:01:06, 5.13it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 6%|▌ | 1175/20000 [05:38<1:01:06, 5.13it/s, lr: 7.0e-05 loss: 1.757e-01] | |
| Saving at step 1175 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001175.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▌ | 1176/20000 [05:39<1:01:06, 5.13it/s, lr: 7.0e-05 loss: 2.718e-01] jam: 6%|▌ | 1177/20000 [05:39<1:31:48, 3.42it/s, lr: 7.0e-05 loss: 2.718e-01] jam: 6%|▌ | 1177/20000 [05:39<1:31:48, 3.42it/s, lr: 7.0e-05 loss: 1.688e-01] jam: 6%|▌ | 1178/20000 [05:39<1:16:48, 4.08it/s, lr: 7.0e-05 loss: 1.688e-01] jam: 6%|▌ | 1178/20000 [05:39<1:16:48, 4.08it/s, lr: 7.0e-05 loss: 2.061e-01] jam: 6%|▌ | 1179/20000 [05:39<1:12:54, 4.30it/s, lr: 7.0e-05 loss: 2.061e-01] jam: 6%|▌ | 1179/20000 [05:40<1:12:54, 4.30it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 6%|▌ | 1180/20000 [05:40<1:37:52, 3.20it/s, lr: 7.0e-05 loss: 1.473e-01] jam: 6%|▌ | 1180/20000 [05:40<1:37:52, 3.20it/s, lr: 7.0e-05 loss: 5.478e-02] jam: 6%|▌ | 1181/20000 [05:40<1:52:12, 2.80it/s, lr: 7.0e-05 loss: 5.478e-02] jam: 6%|▌ | 1181/20000 [05:40<1:52:12, 2.80it/s, lr: 7.0e-05 loss: 1.535e-01] jam: 6%|▌ | 1182/20000 [05:40<1:13:57, 4.24it/s, lr: 7.0e-05 loss: 1.535e-01] jam: 6%|▌ | 1182/20000 [05:41<1:13:57, 4.24it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 6%|▌ | 1183/20000 [05:41<1:10:11, 4.47it/s, lr: 7.0e-05 loss: 2.191e-01] jam: 6%|▌ | 1183/20000 [05:41<1:10:11, 4.47it/s, lr: 7.0e-05 loss: 1.302e-01] jam: 6%|▌ | 1184/20000 [05:41<1:00:21, 5.20it/s, lr: 7.0e-05 loss: 1.302e-01] jam: 6%|▌ | 1184/20000 [05:41<1:00:21, 5.20it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 6%|▌ | 1185/20000 [05:41<1:00:27, 5.19it/s, lr: 7.0e-05 loss: 2.070e-01] jam: 6%|▌ | 1185/20000 [05:41<1:00:27, 5.19it/s, lr: 7.0e-05 loss: 5.286e-02] jam: 6%|▌ | 1186/20000 [05:41<1:33:21, 3.36it/s, lr: 7.0e-05 loss: 5.286e-02] jam: 6%|▌ | 1186/20000 [05:41<1:33:21, 3.36it/s, lr: 7.0e-05 loss: 1.493e-01] jam: 6%|▌ | 1187/20000 [05:41<1:05:47, 4.77it/s, lr: 7.0e-05 loss: 1.493e-01] jam: 6%|▌ | 1187/20000 [05:42<1:05:47, 4.77it/s, lr: 7.0e-05 loss: 2.194e-01] jam: 6%|▌ | 1188/20000 [05:42<1:02:25, 5.02it/s, lr: 7.0e-05 loss: 2.194e-01] jam: 6%|▌ | 1188/20000 [05:42<1:02:25, 5.02it/s, lr: 7.0e-05 loss: 2.175e-01] jam: 6%|▌ | 1189/20000 [05:42<1:33:10, 3.36it/s, lr: 7.0e-05 loss: 2.175e-01] jam: 6%|▌ | 1189/20000 [05:42<1:33:10, 3.36it/s, lr: 7.0e-05 loss: 1.778e-01] jam: 6%|▌ | 1190/20000 [05:42<1:18:20, 4.00it/s, lr: 7.0e-05 loss: 1.778e-01] jam: 6%|▌ | 1190/20000 [05:42<1:18:20, 4.00it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 6%|▌ | 1191/20000 [05:42<1:21:49, 3.83it/s, lr: 7.0e-05 loss: 1.477e-01] jam: 6%|▌ | 1191/20000 [05:43<1:21:49, 3.83it/s, lr: 7.0e-05 loss: 6.175e-02] jam: 6%|▌ | 1192/20000 [05:43<1:38:59, 3.17it/s, lr: 7.0e-05 loss: 6.175e-02] jam: 6%|▌ | 1192/20000 [05:43<1:38:59, 3.17it/s, lr: 7.0e-05 loss: 6.988e-02] jam: 6%|▌ | 1193/20000 [05:43<1:18:08, 4.01it/s, lr: 7.0e-05 loss: 6.988e-02] jam: 6%|▌ | 1193/20000 [05:43<1:18:08, 4.01it/s, lr: 7.0e-05 loss: 1.548e-01] jam: 6%|▌ | 1194/20000 [05:43<1:39:15, 3.16it/s, lr: 7.0e-05 loss: 1.548e-01] jam: 6%|▌ | 1194/20000 [05:44<1:39:15, 3.16it/s, lr: 7.0e-05 loss: 1.653e-01] jam: 6%|▌ | 1195/20000 [05:44<1:51:56, 2.80it/s, lr: 7.0e-05 loss: 1.653e-01] jam: 6%|▌ | 1195/20000 [05:44<1:51:56, 2.80it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 6%|▌ | 1196/20000 [05:44<1:13:35, 4.26it/s, lr: 7.0e-05 loss: 1.088e-01] jam: 6%|▌ | 1196/20000 [05:44<1:13:35, 4.26it/s, lr: 7.0e-05 loss: 1.716e-01] jam: 6%|▌ | 1197/20000 [05:44<1:05:40, 4.77it/s, lr: 7.0e-05 loss: 1.716e-01] jam: 6%|▌ | 1197/20000 [05:44<1:05:40, 4.77it/s, lr: 7.0e-05 loss: 7.926e-02] jam: 6%|▌ | 1198/20000 [05:44<1:34:33, 3.31it/s, lr: 7.0e-05 loss: 7.926e-02] jam: 6%|▌ | 1198/20000 [05:45<1:34:33, 3.31it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 6%|▌ | 1199/20000 [05:45<1:15:29, 4.15it/s, lr: 7.0e-05 loss: 1.442e-01] jam: 6%|▌ | 1199/20000 [05:45<1:15:29, 4.15it/s, lr: 7.0e-05 loss: 1.319e-01] jam: 6%|▌ | 1200/20000 [05:45<1:42:16, 3.06it/s, lr: 7.0e-05 loss: 1.319e-01] jam: 6%|▌ | 1200/20000 [05:45<1:42:16, 3.06it/s, lr: 7.0e-05 loss: 1.048e-01] | |
| Saving at step 1200 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001200.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| jam: 6%|▌ | 1201/20000 [05:46<1:42:16, 3.06it/s, lr: 7.0e-05 loss: 2.186e-01] jam: 6%|▌ | 1202/20000 [05:46<1:21:54, 3.82it/s, lr: 7.0e-05 loss: 2.186e-01] jam: 6%|▌ | 1202/20000 [05:46<1:21:54, 3.82it/s, lr: 7.0e-05 loss: 1.581e-01] jam: 6%|▌ | 1203/20000 [05:46<1:08:17, 4.59it/s, lr: 7.0e-05 loss: 1.581e-01] jam: 6%|▌ | 1203/20000 [05:46<1:08:17, 4.59it/s, lr: 7.0e-05 loss: 2.523e-02] jam: 6%|▌ | 1204/20000 [05:46<1:36:04, 3.26it/s, lr: 7.0e-05 loss: 2.523e-02] jam: 6%|▌ | 1204/20000 [05:47<1:36:04, 3.26it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 6%|▌ | 1205/20000 [05:47<1:51:02, 2.82it/s, lr: 7.0e-05 loss: 1.328e-01] jam: 6%|▌ | 1205/20000 [05:47<1:51:02, 2.82it/s, lr: 7.0e-05 loss: 9.406e-02] jam: 6%|▌ | 1206/20000 [05:47<1:47:34, 2.91it/s, lr: 7.0e-05 loss: 9.406e-02] jam: 6%|▌ | 1206/20000 [05:47<1:47:34, 2.91it/s, lr: 7.0e-05 loss: 1.239e-01] jam: 6%|▌ | 1207/20000 [05:47<1:19:17, 3.95it/s, lr: 7.0e-05 loss: 1.239e-01] jam: 6%|▌ | 1207/20000 [05:47<1:19:17, 3.95it/s, lr: 7.0e-05 loss: 1.120e-01] jam: 6%|▌ | 1208/20000 [05:47<1:13:56, 4.24it/s, lr: 7.0e-05 loss: 1.120e-01] jam: 6%|▌ | 1208/20000 [05:48<1:13:56, 4.24it/s, lr: 7.0e-05 loss: 1.404e-01] jam: 6%|▌ | 1209/20000 [05:48<1:37:16, 3.22it/s, lr: 7.0e-05 loss: 1.404e-01] jam: 6%|▌ | 1209/20000 [05:48<1:37:16, 3.22it/s, lr: 7.0e-05 loss: 2.600e-01] jam: 6%|▌ | 1210/20000 [05:48<1:18:21, 4.00it/s, lr: 7.0e-05 loss: 2.600e-01] jam: 6%|▌ | 1210/20000 [05:48<1:18:21, 4.00it/s, lr: 7.0e-05 loss: 3.429e-01] jam: 6%|▌ | 1211/20000 [05:48<1:43:22, 3.03it/s, lr: 7.0e-05 loss: 3.429e-01] jam: 6%|▌ | 1211/20000 [05:49<1:43:22, 3.03it/s, lr: 7.0e-05 loss: 2.118e-01] jam: 6%|▌ | 1212/20000 [05:49<1:45:12, 2.98it/s, lr: 7.0e-05 loss: 2.118e-01] jam: 6%|▌ | 1212/20000 [05:49<1:45:12, 2.98it/s, lr: 7.0e-05 loss: 2.448e-01] jam: 6%|▌ | 1213/20000 [05:49<1:52:41, 2.78it/s, lr: 7.0e-05 loss: 2.448e-01] jam: 6%|▌ | 1213/20000 [05:49<1:52:41, 2.78it/s, lr: 7.0e-05 loss: 1.676e-01] jam: 6%|▌ | 1214/20000 [05:49<1:55:19, 2.71it/s, lr: 7.0e-05 loss: 1.676e-01] jam: 6%|▌ | 1214/20000 [05:50<1:55:19, 2.71it/s, lr: 7.0e-05 loss: 1.597e-01] jam: 6%|▌ | 1215/20000 [05:50<1:48:59, 2.87it/s, lr: 7.0e-05 loss: 1.597e-01] jam: 6%|▌ | 1215/20000 [05:50<1:48:59, 2.87it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 6%|▌ | 1216/20000 [05:50<1:20:49, 3.87it/s, lr: 7.0e-05 loss: 1.580e-01] jam: 6%|▌ | 1216/20000 [05:50<1:20:49, 3.87it/s, lr: 7.0e-05 loss: 5.643e-02] jam: 6%|▌ | 1217/20000 [05:50<1:11:31, 4.38it/s, lr: 7.0e-05 loss: 5.643e-02] jam: 6%|▌ | 1217/20000 [05:50<1:11:31, 4.38it/s, lr: 7.0e-05 loss: 2.459e-01] jam: 6%|▌ | 1218/20000 [05:50<1:07:16, 4.65it/s, lr: 7.0e-05 loss: 2.459e-01] jam: 6%|▌ | 1218/20000 [05:51<1:07:16, 4.65it/s, lr: 7.0e-05 loss: 1.881e-01] jam: 6%|▌ | 1219/20000 [05:51<1:04:44, 4.84it/s, lr: 7.0e-05 loss: 1.881e-01] jam: 6%|▌ | 1219/20000 [05:51<1:04:44, 4.84it/s, lr: 7.0e-05 loss: 1.768e-01] jam: 6%|▌ | 1220/20000 [05:51<1:34:52, 3.30it/s, lr: 7.0e-05 loss: 1.768e-01] jam: 6%|▌ | 1220/20000 [05:51<1:34:52, 3.30it/s, lr: 7.0e-05 loss: 5.741e-02] jam: 6%|▌ | 1221/20000 [05:51<1:50:21, 2.84it/s, lr: 7.0e-05 loss: 5.741e-02] jam: 6%|▌ | 1221/20000 [05:52<1:50:21, 2.84it/s, lr: 7.0e-05 loss: 1.396e-01] jam: 6%|▌ | 1222/20000 [05:52<1:14:00, 4.23it/s, lr: 7.0e-05 loss: 1.396e-01] jam: 6%|▌ | 1222/20000 [05:52<1:14:00, 4.23it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 6%|▌ | 1223/20000 [05:52<1:36:30, 3.24it/s, lr: 7.0e-05 loss: 1.634e-01] jam: 6%|▌ | 1223/20000 [05:52<1:36:30, 3.24it/s, lr: 7.0e-05 loss: 2.125e-01] jam: 6%|▌ | 1224/20000 [05:52<1:19:33, 3.93it/s, lr: 7.0e-05 loss: 2.125e-01] jam: 6%|▌ | 1224/20000 [05:52<1:19:33, 3.93it/s, lr: 7.0e-05 loss: 1.847e-01] jam: 6%|▌ | 1225/20000 [05:52<1:41:00, 3.10it/s, lr: 7.0e-05 loss: 1.847e-01] jam: 6%|▌ | 1225/20000 [05:53<1:41:00, 3.10it/s, lr: 7.0e-05 loss: 1.541e-01] | |
| Saving at step 1225 | |
| Saved checkpoint to /app/ai-toolkit/output/jam/jam_000001225.safetensors | |
| Saved optimizer to /app/ai-toolkit/output/jam/optimizer.pt | |
| ======================================== | |
| Job stopped | |
| ======================================== | |
Xet Storage Details
- Size:
- 230 kB
- Xet hash:
- 921cd9c0743773f20cad1285ab8ae999a0d0deed3adb8e34a096ca8606247e69
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.