| { |
| "wall_seconds": 60962.09582400322, |
| "records_seen": 7078, |
| "requested_records": 7078, |
| "truncated_records": 0, |
| "rejected_records": 0, |
| "optimizer_steps": 885, |
| "forward_tokens": 4276019, |
| "step_seconds": [ |
| 96.887, |
| 130.239, |
| 64.843, |
| 118.489, |
| 72.557, |
| 131.936, |
| 87.633, |
| 65.411, |
| 141.035, |
| 61.861, |
| 96.225, |
| 55.674, |
| 101.937, |
| 56.589, |
| 92.727, |
| 89.455, |
| 71.768, |
| 105.296, |
| 47.594, |
| 74.27, |
| 86.787, |
| 137.428, |
| 66.639, |
| 62.426, |
| 68.465, |
| 90.867, |
| 39.176, |
| 88.074, |
| 74.488, |
| 125.688, |
| 67.986, |
| 182.398, |
| 48.921, |
| 54.761, |
| 60.658, |
| 119.669, |
| 176.779, |
| 54.092, |
| 62.95, |
| 100.736, |
| 80.941, |
| 52.29, |
| 115.236, |
| 36.232, |
| 73.326, |
| 73.321, |
| 68.473, |
| 45.38, |
| 83.677, |
| 87.527, |
| 52.692, |
| 86.94, |
| 75.394, |
| 51.033, |
| 87.929, |
| 81.854, |
| 76.408, |
| 98.191, |
| 26.667, |
| 119.873, |
| 125.848, |
| 28.405, |
| 79.938, |
| 56.386, |
| 145.557, |
| 102.2, |
| 85.957, |
| 155.796, |
| 75.795, |
| 128.731, |
| 81.014, |
| 79.759, |
| 97.159, |
| 84.639, |
| 87.384, |
| 104.541, |
| 33.152, |
| 95.056, |
| 70.181, |
| 83.354, |
| 100.631, |
| 114.43, |
| 51.116, |
| 91.421, |
| 27.912, |
| 68.285, |
| 103.752, |
| 56.595, |
| 69.161, |
| 92.275, |
| 56.924, |
| 64.621, |
| 45.712, |
| 179.528, |
| 73.974, |
| 50.731, |
| 74.07, |
| 97.018, |
| 122.123, |
| 28.387, |
| 60.449, |
| 124.632, |
| 121.305, |
| 69.911, |
| 31.255, |
| 61.375, |
| 46.768, |
| 77.494, |
| 84.681, |
| 98.937, |
| 52.35, |
| 35.93, |
| 183.907, |
| 49.171, |
| 61.316, |
| 109.907, |
| 98.022, |
| 78.431, |
| 102.459, |
| 83.348, |
| 45.117, |
| 101.874, |
| 35.329, |
| 152.5, |
| 30.742, |
| 111.929, |
| 129.277, |
| 131.915, |
| 79.439, |
| 185.866, |
| 90.869, |
| 88.047, |
| 92.784, |
| 62.367, |
| 74.109, |
| 94.238, |
| 99.895, |
| 136.447, |
| 95.391, |
| 79.64, |
| 57.52, |
| 107.161, |
| 38.442, |
| 80.935, |
| 83.833, |
| 161.033, |
| 49.887, |
| 98.704, |
| 66.615, |
| 99.616, |
| 220.619, |
| 106.419, |
| 120.217, |
| 79.101, |
| 92.917, |
| 59.415, |
| 64.718, |
| 45.108, |
| 226.587, |
| 88.654, |
| 86.273, |
| 111.679, |
| 79.214, |
| 62.438, |
| 68.535, |
| 31.884, |
| 61.365, |
| 113.21, |
| 66.201, |
| 79.717, |
| 86.644, |
| 59.071, |
| 92.64, |
| 61.596, |
| 204.354, |
| 104.706, |
| 100.352, |
| 134.343, |
| 67.505, |
| 38.503, |
| 120.369, |
| 92.993, |
| 63.236, |
| 66.123, |
| 87.475, |
| 73.423, |
| 174.317, |
| 74.719, |
| 97.19, |
| 60.986, |
| 216.583, |
| 98.451, |
| 249.661, |
| 74.652, |
| 71.156, |
| 59.036, |
| 82.632, |
| 65.212, |
| 84.793, |
| 39.748, |
| 61.606, |
| 78.421, |
| 85.292, |
| 94.673, |
| 117.271, |
| 87.963, |
| 104.816, |
| 64.627, |
| 73.091, |
| 55.52, |
| 83.14, |
| 102.323, |
| 112.942, |
| 53.369, |
| 142.908, |
| 113.983, |
| 162.186, |
| 57.785, |
| 55.968, |
| 103.319, |
| 98.048, |
| 74.754, |
| 58.825, |
| 86.357, |
| 47.483, |
| 88.856, |
| 100.018, |
| 92.093, |
| 109.705, |
| 71.624, |
| 77.28, |
| 37.432, |
| 56.82, |
| 82.718, |
| 87.899, |
| 95.711, |
| 72.451, |
| 51.838, |
| 65.507, |
| 115.124, |
| 63.418, |
| 67.584, |
| 97.552, |
| 100.733, |
| 51.72, |
| 133.577, |
| 67.492, |
| 71.278, |
| 80.782, |
| 72.044, |
| 88.127, |
| 63.45, |
| 64.824, |
| 110.97, |
| 109.481, |
| 59.442, |
| 65.538, |
| 63.509, |
| 96.284, |
| 87.036, |
| 118.24, |
| 93.737, |
| 86.733, |
| 91.856, |
| 73.546, |
| 63.268, |
| 87.71, |
| 59.071, |
| 93.523, |
| 111.603, |
| 120.382, |
| 90.223, |
| 101.585, |
| 60.685, |
| 159.22, |
| 181.41, |
| 87.87, |
| 68.945, |
| 134.911, |
| 113.442, |
| 43.799, |
| 96.309, |
| 88.036, |
| 117.504, |
| 113.94, |
| 89.087, |
| 123.215, |
| 88.196, |
| 85.822, |
| 112.495, |
| 91.786, |
| 80.933, |
| 103.731, |
| 81.863, |
| 164.1, |
| 57.882, |
| 76.102, |
| 69.184, |
| 85.967, |
| 48.241, |
| 66.663, |
| 111.394, |
| 88.121, |
| 40.214, |
| 46.355, |
| 52.812, |
| 94.952, |
| 68.097, |
| 73.143, |
| 45.43, |
| 71.362, |
| 97.048, |
| 27.802, |
| 83.727, |
| 55.881, |
| 92.733, |
| 81.312, |
| 45.27, |
| 107.173, |
| 125.598, |
| 76.064, |
| 113.016, |
| 86.96, |
| 127.539, |
| 43.926, |
| 42.319, |
| 91.346, |
| 153.426, |
| 74.188, |
| 82.735, |
| 50.591, |
| 80.364, |
| 80.894, |
| 80.227, |
| 26.713, |
| 73.932, |
| 104.28, |
| 56.079, |
| 69.111, |
| 136.948, |
| 57.417, |
| 100.784, |
| 82.223, |
| 69.2, |
| 80.888, |
| 73.496, |
| 69.842, |
| 78.814, |
| 72.03, |
| 61.726, |
| 81.737, |
| 56.325, |
| 79.539, |
| 68.088, |
| 63.187, |
| 36.016, |
| 51.547, |
| 90.949, |
| 66.929, |
| 44.926, |
| 111.053, |
| 74.073, |
| 81.871, |
| 55.728, |
| 72.646, |
| 94.364, |
| 113.075, |
| 55.23, |
| 38.396, |
| 74.463, |
| 52.253, |
| 91.838, |
| 67.789, |
| 179.379, |
| 101.791, |
| 97.319, |
| 138.766, |
| 50.958, |
| 84.461, |
| 43.635, |
| 58.504, |
| 25.198, |
| 73.031, |
| 82.066, |
| 64.344, |
| 54.417, |
| 120.191, |
| 74.743, |
| 108.504, |
| 60.701, |
| 100.233, |
| 78.954, |
| 27.931, |
| 106.766, |
| 25.496, |
| 109.934, |
| 88.142, |
| 179.485, |
| 73.894, |
| 92.787, |
| 30.693, |
| 93.312, |
| 82.278, |
| 69.303, |
| 31.419, |
| 86.561, |
| 151.524, |
| 30.109, |
| 69.535, |
| 37.953, |
| 64.512, |
| 76.458, |
| 119.842, |
| 73.943, |
| 43.945, |
| 58.117, |
| 59.338, |
| 73.969, |
| 93.951, |
| 64.211, |
| 41.59, |
| 24.379, |
| 93.928, |
| 116.444, |
| 43.856, |
| 92.366, |
| 55.073, |
| 19.17, |
| 66.899, |
| 55.8, |
| 44.017, |
| 98.014, |
| 115.18, |
| 66.85, |
| 72.858, |
| 46.234, |
| 78.499, |
| 47.289, |
| 43.999, |
| 74.629, |
| 61.054, |
| 70.622, |
| 52.574, |
| 14.414, |
| 35.659, |
| 38.241, |
| 60.782, |
| 84.859, |
| 85.475, |
| 34.745, |
| 98.385, |
| 60.112, |
| 80.424, |
| 62.163, |
| 39.213, |
| 64.45, |
| 63.212, |
| 50.869, |
| 69.918, |
| 74.74, |
| 41.65, |
| 69.487, |
| 36.575, |
| 59.245, |
| 38.485, |
| 43.856, |
| 72.132, |
| 19.146, |
| 62.713, |
| 68.039, |
| 52.139, |
| 59.647, |
| 50.036, |
| 86.751, |
| 32.585, |
| 53.94, |
| 23.154, |
| 16.498, |
| 122.387, |
| 87.725, |
| 109.58, |
| 32.614, |
| 38.39, |
| 49.632, |
| 63.778, |
| 54.399, |
| 68.662, |
| 48.935, |
| 39.172, |
| 26.178, |
| 68.163, |
| 41.545, |
| 96.49, |
| 56.105, |
| 31.043, |
| 33.809, |
| 60.326, |
| 36.061, |
| 28.817, |
| 70.438, |
| 55.613, |
| 86.366, |
| 101.145, |
| 78.433, |
| 52.84, |
| 72.17, |
| 64.271, |
| 72.391, |
| 86.575, |
| 130.558, |
| 59.331, |
| 54.996, |
| 19.211, |
| 53.388, |
| 51.521, |
| 93.414, |
| 50.071, |
| 75.611, |
| 96.365, |
| 45.114, |
| 61.443, |
| 40.618, |
| 50.441, |
| 56.784, |
| 50.535, |
| 117.912, |
| 75.757, |
| 81.528, |
| 76.297, |
| 44.461, |
| 69.583, |
| 65.725, |
| 17.908, |
| 33.368, |
| 53.098, |
| 67.286, |
| 46.546, |
| 37.99, |
| 43.876, |
| 82.532, |
| 31.879, |
| 50.029, |
| 67.075, |
| 48.093, |
| 40.986, |
| 57.201, |
| 65.138, |
| 38.782, |
| 62.556, |
| 56.224, |
| 121.114, |
| 59.417, |
| 97.831, |
| 31.901, |
| 52.621, |
| 48.209, |
| 21.907, |
| 64.196, |
| 86.728, |
| 130.048, |
| 39.063, |
| 60.024, |
| 58.436, |
| 45.132, |
| 28.697, |
| 29.617, |
| 49.493, |
| 46.495, |
| 57.513, |
| 85.637, |
| 69.675, |
| 38.817, |
| 44.946, |
| 46.659, |
| 51.017, |
| 43.054, |
| 66.946, |
| 67.982, |
| 52.015, |
| 47.246, |
| 116.342, |
| 57.275, |
| 50.755, |
| 40.371, |
| 41.278, |
| 65.988, |
| 63.794, |
| 39.48, |
| 37.646, |
| 35.44, |
| 59.309, |
| 44.092, |
| 91.05, |
| 31.293, |
| 79.705, |
| 60.88, |
| 43.093, |
| 50.531, |
| 58.631, |
| 62.754, |
| 62.182, |
| 54.619, |
| 46.622, |
| 93.471, |
| 40.252, |
| 122.889, |
| 76.653, |
| 57.969, |
| 98.75, |
| 86.485, |
| 61.187, |
| 64.951, |
| 33.246, |
| 71.691, |
| 73.062, |
| 32.862, |
| 56.818, |
| 23.651, |
| 31.307, |
| 32.262, |
| 43.5, |
| 45.742, |
| 13.262, |
| 38.091, |
| 55.787, |
| 49.842, |
| 29.699, |
| 43.865, |
| 72.126, |
| 54.191, |
| 31.41, |
| 40.318, |
| 51.776, |
| 63.911, |
| 25.619, |
| 21.098, |
| 16.837, |
| 85.729, |
| 74.337, |
| 52.411, |
| 54.995, |
| 65.253, |
| 57.833, |
| 35.986, |
| 57.441, |
| 64.145, |
| 39.398, |
| 51.665, |
| 85.247, |
| 43.665, |
| 45.633, |
| 38.602, |
| 29.115, |
| 63.166, |
| 51.115, |
| 90.545, |
| 36.578, |
| 44.682, |
| 29.098, |
| 25.297, |
| 52.218, |
| 24.776, |
| 51.328, |
| 64.042, |
| 39.146, |
| 48.733, |
| 42.213, |
| 24.194, |
| 42.32, |
| 64.64, |
| 48.365, |
| 51.991, |
| 65.944, |
| 55.765, |
| 64.998, |
| 47.689, |
| 33.648, |
| 72.752, |
| 53.651, |
| 39.306, |
| 69.388, |
| 65.299, |
| 55.998, |
| 52.91, |
| 51.839, |
| 53.079, |
| 67.106, |
| 44.509, |
| 46.101, |
| 54.223, |
| 47.013, |
| 49.686, |
| 31.62, |
| 59.542, |
| 34.3, |
| 102.609, |
| 31.538, |
| 72.437, |
| 63.499, |
| 48.811, |
| 36.443, |
| 24.429, |
| 71.636, |
| 49.852, |
| 42.319, |
| 72.06, |
| 83.845, |
| 49.052, |
| 58.731, |
| 53.477, |
| 120.86, |
| 58.412, |
| 26.997, |
| 65.748, |
| 46.7, |
| 93.338, |
| 67.989, |
| 42.664, |
| 54.86, |
| 62.672, |
| 44.329, |
| 58.656, |
| 17.285, |
| 71.767, |
| 60.339, |
| 40.405, |
| 55.516, |
| 39.399, |
| 40.745, |
| 44.258, |
| 42.121, |
| 83.306, |
| 76.928, |
| 27.378, |
| 37.235, |
| 38.145, |
| 49.338, |
| 41.528, |
| 82.741, |
| 102.42, |
| 32.067, |
| 41.59, |
| 46.331, |
| 94.141, |
| 122.677, |
| 28.809, |
| 58.918, |
| 59.139, |
| 34.514, |
| 50.713, |
| 47.028, |
| 37.277, |
| 32.581, |
| 37.146, |
| 72.023, |
| 82.423, |
| 43.047, |
| 61.21, |
| 44.923, |
| 86.05, |
| 56.766, |
| 45.187, |
| 57.055, |
| 45.346, |
| 58.585, |
| 69.061, |
| 41.551, |
| 32.341, |
| 56.401, |
| 46.737, |
| 63.715, |
| 48.138, |
| 50.84, |
| 48.901, |
| 17.771, |
| 32.771, |
| 59.071, |
| 120.378, |
| 28.196, |
| 64.42, |
| 34.867, |
| 46.681, |
| 65.969, |
| 113.467, |
| 51.2, |
| 66.156, |
| 49.362, |
| 53.608, |
| 32.013, |
| 65.117, |
| 37.012, |
| 49.484, |
| 43.924, |
| 48.982, |
| 63.861, |
| 51.029, |
| 34.369, |
| 58.41, |
| 31.521, |
| 53.407, |
| 97.715, |
| 55.667, |
| 45.741, |
| 22.709, |
| 66.862, |
| 35.728, |
| 85.854, |
| 33.46, |
| 29.56, |
| 40.87, |
| 40.286, |
| 30.118, |
| 62.325, |
| 26.149, |
| 24.105, |
| 52.797, |
| 57.236, |
| 42.192, |
| 55.821, |
| 31.128, |
| 26.643, |
| 37.13, |
| 16.883, |
| 61.222, |
| 84.193, |
| 31.288, |
| 35.285, |
| 54.858, |
| 61.679, |
| 44.614, |
| 45.801, |
| 31.422, |
| 53.811, |
| 51.264, |
| 98.162, |
| 57.921, |
| 59.463, |
| 44.425, |
| 50.847, |
| 71.173, |
| 51.704, |
| 33.639, |
| 65.328, |
| 33.071, |
| 42.4, |
| 24.335, |
| 32.753, |
| 48.663, |
| 51.392, |
| 45.763, |
| 54.92, |
| 53.119, |
| 43.084, |
| 29.486, |
| 26.604, |
| 67.864, |
| 61.957, |
| 58.881, |
| 48.405, |
| 94.57, |
| 32.319, |
| 35.866, |
| 49.71, |
| 46.075, |
| 71.016, |
| 45.339, |
| 43.534, |
| 54.123, |
| 82.674, |
| 31.486, |
| 37.047, |
| 47.222, |
| 36.153, |
| 54.698, |
| 42.938, |
| 65.524, |
| 30.103, |
| 37.382, |
| 44.257, |
| 21.746, |
| 55.954, |
| 65.433, |
| 53.838, |
| 55.63, |
| 18.29 |
| ], |
| "optimizer_seconds": 196.13493037223816, |
| "resume_seconds": [ |
| 1.568, |
| 1.82, |
| 1.623, |
| 3.273, |
| 2.216, |
| 1.366, |
| 1.203, |
| 1.029, |
| 0.969, |
| 1.12, |
| 1.124, |
| 0.852, |
| 1.762, |
| 0.884, |
| 1.345, |
| 1.071, |
| 1.182, |
| 1.02, |
| 1.221, |
| 0.943, |
| 0.967, |
| 0.932, |
| 1.132, |
| 0.94 |
| ], |
| "resume_write_seconds": [ |
| 1.3, |
| 1.6, |
| 1.4, |
| 2.5, |
| 1.8, |
| 1.2, |
| 0.9, |
| 0.7, |
| 0.8, |
| 1.0, |
| 0.9, |
| 0.6, |
| 1.6, |
| 0.7, |
| 1.2, |
| 0.9, |
| 1.0, |
| 0.8, |
| 1.0, |
| 0.7, |
| 0.8, |
| 0.7, |
| 1.0, |
| 0.8 |
| ], |
| "backbone_save_seconds": 0.2, |
| "grad_norm": [ |
| { |
| "epoch": 0, |
| "steps": 885, |
| "mean": 54.624889955932204, |
| "max": 16406.365234, |
| "clipped_steps": 869 |
| } |
| ], |
| "nonfinite_skipped": {}, |
| "weights": "lora", |
| "peak_device_bytes": 15790329088, |
| "device": "mps", |
| "dtype": "fp32", |
| "batch": 2, |
| "peak_rss_bytes": 1370767360 |
| } |
|
|
