Upload logs/2026-09-23_t128d_ship.log with huggingface_hub
Browse files
logs/2026-09-23_t128d_ship.log
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[08:24:32] === t128d shipper: rank_0 every save, full set at 300 360 ===
|
| 2 |
+
[08:54:37] shipped t128d_step50_rank0.ckpt (2.6G)
|
| 3 |
+
[09:16:46] shipped t128d_step100_rank0.ckpt (2.6G)
|
| 4 |
+
[09:43:27] shipped t128d_step150_rank0.ckpt (2.6G)
|
| 5 |
+
[10:10:06] shipped t128d_step200_rank0.ckpt (2.6G)
|
| 6 |
+
[10:40:56] shipped t128d_step250_rank0.ckpt (2.6G)
|
| 7 |
+
[11:03:51] shipped t128d_step300_rank0.ckpt (2.6G)
|
| 8 |
+
[11:04:01] shipped t128d_step300_rank0.zero_g1 (590M)
|
| 9 |
+
[11:04:17] shipped t128d_step300_rank1.ckpt (2.6G)
|
| 10 |
+
[11:04:20] shipped t128d_step300_rank1.zero_g1 (320K)
|
| 11 |
+
[11:04:36] shipped t128d_step300_rank2.ckpt (2.6G)
|
| 12 |
+
[11:04:39] shipped t128d_step300_rank2.zero_g1 (324K)
|
| 13 |
+
[11:04:55] shipped t128d_step300_rank3.ckpt (2.6G)
|
| 14 |
+
[11:04:58] shipped t128d_step300_rank3.zero_g1 (324K)
|
| 15 |
+
[11:04:58] MILESTONE 300: full resumable set up
|
| 16 |
+
[11:05:43] MILESTONE 300: full resumable set up
|
| 17 |
+
[11:06:28] MILESTONE 300: full resumable set up
|
| 18 |
+
[11:07:13] MILESTONE 300: full resumable set up
|
| 19 |
+
[11:07:58] MILESTONE 300: full resumable set up
|
| 20 |
+
[11:08:43] MILESTONE 300: full resumable set up
|
| 21 |
+
[11:09:29] MILESTONE 300: full resumable set up
|
| 22 |
+
[11:10:14] MILESTONE 300: full resumable set up
|
| 23 |
+
[11:10:59] MILESTONE 300: full resumable set up
|
| 24 |
+
[11:11:44] MILESTONE 300: full resumable set up
|
| 25 |
+
[11:12:29] MILESTONE 300: full resumable set up
|
| 26 |
+
[11:13:14] MILESTONE 300: full resumable set up
|
| 27 |
+
[11:13:59] MILESTONE 300: full resumable set up
|
| 28 |
+
[11:14:44] MILESTONE 300: full resumable set up
|
| 29 |
+
[11:15:29] MILESTONE 300: full resumable set up
|
| 30 |
+
[11:16:15] MILESTONE 300: full resumable set up
|
| 31 |
+
[11:17:00] MILESTONE 300: full resumable set up
|
| 32 |
+
[11:17:45] MILESTONE 300: full resumable set up
|
| 33 |
+
[11:18:30] MILESTONE 300: full resumable set up
|
| 34 |
+
[11:19:15] MILESTONE 300: full resumable set up
|
| 35 |
+
[11:20:00] MILESTONE 300: full resumable set up
|
| 36 |
+
[11:20:45] MILESTONE 300: full resumable set up
|
| 37 |
+
[11:21:30] MILESTONE 300: full resumable set up
|
| 38 |
+
[11:22:15] MILESTONE 300: full resumable set up
|
| 39 |
+
[11:23:01] MILESTONE 300: full resumable set up
|
| 40 |
+
[11:23:46] MILESTONE 300: full resumable set up
|
| 41 |
+
[11:24:31] MILESTONE 300: full resumable set up
|
| 42 |
+
[11:25:16] MILESTONE 300: full resumable set up
|
| 43 |
+
[11:26:01] MILESTONE 300: full resumable set up
|
| 44 |
+
[11:26:46] MILESTONE 300: full resumable set up
|
| 45 |
+
[11:27:31] MILESTONE 300: full resumable set up
|
| 46 |
+
[11:28:16] MILESTONE 300: full resumable set up
|
| 47 |
+
[11:29:01] MILESTONE 300: full resumable set up
|
| 48 |
+
[11:29:47] MILESTONE 300: full resumable set up
|
| 49 |
+
[11:30:09] shipped t128d_step350_rank0.ckpt (2.6G)
|
| 50 |
+
[11:30:54] MILESTONE 300: full resumable set up
|
| 51 |
+
[11:31:39] MILESTONE 300: full resumable set up
|
| 52 |
+
[11:32:24] MILESTONE 300: full resumable set up
|
| 53 |
+
[11:33:10] MILESTONE 300: full resumable set up
|
| 54 |
+
[11:33:55] MILESTONE 300: full resumable set up
|
| 55 |
+
[11:34:40] MILESTONE 300: full resumable set up
|
| 56 |
+
[11:35:25] MILESTONE 300: full resumable set up
|
| 57 |
+
[11:35:47] shipped t128d_step360_rank0.ckpt (2.6G)
|
| 58 |
+
[11:35:57] shipped t128d_step360_rank0.zero_g1 (590M)
|
| 59 |
+
[11:38:15] shipped t128d_step360_rank1.ckpt (2.6G)
|
| 60 |
+
[11:38:18] shipped t128d_step360_rank1.zero_g1 (320K)
|
| 61 |
+
[11:40:37] shipped t128d_step360_rank2.ckpt (2.6G)
|
| 62 |
+
[11:40:40] shipped t128d_step360_rank2.zero_g1 (324K)
|
| 63 |
+
[11:40:56] shipped t128d_step360_rank3.ckpt (2.6G)
|
| 64 |
+
[11:40:58] shipped t128d_step360_rank3.zero_g1 (324K)
|
| 65 |
+
[11:40:58] MILESTONE 360: full resumable set up
|
| 66 |
+
[11:41:58] === trainer gone; shipped 22 files ===
|