Upload folder using huggingface_hub

logs/csv/CP_Training_log10.csv

@@ -0,0 +1,78 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,11.082465,-1.000000,0.001000,20.514921,6900.954590,75973.257493,12.809984,2537.849171,17.143392,4204.445755,0.003424,90.402466,67414.875000
+1,9.639948,-1.000000,0.001000,39.792797,6531.472168,80271.030254,9.938656,2217.773804,13.664608,4192.452332,0.003744,77.092796,67542.875000
+2,8.866240,-1.000000,0.001000,128.433212,6517.574707,80442.192620,9.929152,2219.347172,13.602592,4176.882858,0.003872,77.262367,67542.875000
+3,8.534672,-1.000000,0.001000,23.686785,6098.651367,85967.858865,9.821312,1803.796776,13.614400,4173.667603,0.003232,77.252029,67542.875000
+4,8.484062,-1.000000,0.001000,72.269531,6104.810547,85881.125380,9.792160,1805.825340,13.607104,4177.838074,0.003168,77.258499,67542.875000
+5,8.313579,-1.000000,0.001000,13.841644,6127.431641,85564.071662,9.931488,1809.903385,13.633504,4196.302048,0.003840,77.208290,67542.875000
+6,8.156507,-1.000000,0.001000,13.804612,6137.777344,85419.846736,9.915680,1811.848057,13.609152,4204.421463,0.003520,77.483650,67542.875000
+7,7.942274,-1.000000,0.001000,11.322829,6141.116211,85373.404767,9.920800,1813.747353,13.665664,4205.545242,0.003168,77.622238,67542.875000
+8,7.792223,-1.000000,0.001000,23.193489,6152.145996,85220.344305,9.837984,1815.963062,13.656896,4214.662109,0.003200,77.348991,67542.875000
+9,7.995893,-1.000000,0.001000,20.272591,6158.444336,85133.188091,9.960960,1816.925430,13.688000,4220.006424,0.003744,77.321663,67542.875000
+10,7.836282,-1.000000,0.001000,10.287439,6165.194824,85039.972774,9.975808,1818.901154,13.642176,4224.245529,0.003552,77.859009,67542.875000
+11,7.573073,-1.000000,0.001000,9.343869,6173.072754,84931.446769,9.912544,1821.202587,13.690528,4229.890320,0.003808,77.689667,67542.875000
+12,7.572920,-1.000000,0.001000,11.004880,6176.150391,84889.124591,9.988288,1822.820801,13.690144,4231.147568,0.003328,77.602943,67542.875000
+13,7.387016,-1.000000,0.001000,7.115844,6174.544434,84911.203675,9.905984,1824.307137,13.748000,4228.620407,0.003776,77.318176,67542.875000
+14,7.252840,-1.000000,0.001000,6.461637,6181.321289,84818.111773,9.949664,1825.911106,13.681888,4233.754868,0.003040,77.411011,67542.875000
+15,7.277268,-1.000000,0.001000,7.854430,6186.117676,84752.348319,9.889440,1827.270760,13.691328,4236.959152,0.003744,77.698814,67542.875000
+16,7.147101,-1.000000,0.001000,3.713808,6192.219727,84668.830105,9.979136,1829.446724,13.683744,4240.834518,0.003648,77.680702,67542.875000
+17,7.073318,-1.000000,0.000999,3.915107,6196.917969,84604.637764,10.015744,1830.279102,13.702944,4244.546890,0.003392,77.562080,67542.875000
+18,6.967671,-1.000000,0.000999,5.624188,6198.271484,84586.162662,10.071392,1830.439709,13.751008,4245.417206,0.003296,77.996445,67542.875000
+19,7.003613,-1.000000,0.000999,6.782012,6205.808594,84483.430657,9.930144,1832.040573,13.695936,4251.799011,0.003808,77.722626,67542.875000
+20,6.929024,-1.000000,0.000999,2.929501,6204.151855,84505.990861,9.920928,1832.747208,13.702784,4249.210297,0.003840,77.857407,67542.875000
+21,6.853117,-1.000000,0.000999,1.563694,6205.301270,84490.337733,9.899808,1834.452068,13.706368,4249.116074,0.003296,77.441437,67542.875000
+22,6.882618,-1.000000,0.000999,1.999874,6213.014648,84385.444050,10.092800,1835.803265,13.695520,4255.236206,0.003520,77.531837,67542.875000
+23,6.791714,-1.000000,0.000999,4.164785,6211.601074,84404.647648,10.028032,1835.479778,13.788992,4253.619476,0.003552,77.968002,67542.875000
+24,6.775381,-1.000000,0.000999,3.666106,6218.120117,84316.158279,10.070816,1837.318501,13.709088,4257.953934,0.003424,78.434433,67542.875000
+25,6.714929,-1.000000,0.000999,2.181702,6212.209961,84396.374768,9.896128,1836.147972,13.740192,4253.341232,0.003744,78.303421,67542.875000
+26,6.623782,-1.000000,0.000999,1.501676,6218.517090,84310.775773,10.023008,1837.075546,13.744384,4258.935181,0.003232,78.050339,67542.875000
+27,6.509890,-1.000000,0.000999,1.502595,6221.229004,84274.023617,10.091200,1838.834572,13.700704,4259.738983,0.003200,78.243645,67542.875000
+28,6.549478,-1.000000,0.000998,1.444647,6219.298340,84300.184900,9.995488,1838.744385,13.686624,4258.213165,0.003232,77.989792,67542.875000
+29,6.542458,-1.000000,0.000998,0.958420,6221.432617,84271.265520,10.021696,1839.986755,13.769280,4259.424057,0.003168,77.594879,67542.875000
+30,6.537173,-1.000000,0.000998,0.880525,6218.460938,84311.537094,9.866560,1838.281029,13.706272,4258.547699,0.003392,77.330338,67542.875000
+31,6.444611,-1.000000,0.000998,0.942914,6222.914062,84251.203654,9.985344,1839.949242,13.721760,4261.160416,0.003776,77.465378,67542.875000
+32,6.465725,-1.000000,0.000998,1.160179,6225.018555,84222.720847,10.194368,1838.718559,13.710656,4263.817581,0.003616,77.978462,67542.875000
+33,6.335793,-1.000000,0.000998,1.028369,6227.765137,84185.576766,10.014656,1841.263012,13.721120,4264.690918,0.003200,77.403297,67542.875000
+34,6.476894,-1.000000,0.000998,0.955569,6228.075195,84181.385670,9.906752,1841.311474,13.749728,4264.717010,0.003808,77.704674,67542.875000
+35,6.437809,-1.000000,0.000997,1.450145,6223.812988,84239.034975,10.031264,1840.426498,13.830336,4261.140457,0.003392,77.761124,67542.875000
+36,6.384038,-1.000000,0.000997,1.844744,6223.884277,84238.070092,10.065408,1840.901489,13.736736,4260.940277,0.003488,77.520195,67542.875000
+37,6.340375,-1.000000,0.000997,1.126358,6229.255859,84165.430324,10.074048,1840.473347,13.762624,4266.542267,0.003776,77.754784,67542.875000
+38,6.306479,-1.000000,0.000997,0.647389,6230.795410,84144.634110,10.090848,1843.017174,13.782272,4265.629059,0.003360,77.457184,67542.875000
+39,6.293424,-1.000000,0.000997,0.681156,6230.011719,84155.218909,10.032704,1842.463242,13.784864,4264.939117,0.003648,78.052925,67542.875000
+40,6.247808,-1.000000,0.000997,0.766744,6227.478027,84189.458027,9.953760,1841.682117,13.736224,4263.739380,0.003680,77.635330,67542.875000
+41,6.282911,-1.000000,0.000996,0.846366,6230.332520,84150.885744,10.172480,1842.776348,13.905216,4264.593994,0.003808,78.012192,67542.875000
+42,6.259625,-1.000000,0.000996,0.827145,6234.076172,84100.351928,10.028480,1843.140831,13.786528,4268.426056,0.003872,77.797951,67542.875000
+43,6.224118,-1.000000,0.000996,0.527271,6231.372559,84136.840651,10.071744,1842.712696,13.823232,4266.276505,0.003680,77.653694,67542.875000
+44,6.171274,-1.000000,0.000996,0.530845,6234.269531,84097.743508,10.175712,1843.963837,13.870784,4267.498032,0.003712,77.904480,67542.875000
+45,6.256998,-1.000000,0.000996,0.674320,6230.862305,84143.730733,9.981920,1841.723549,13.837600,4266.704086,0.003200,77.814621,67542.875000
+46,6.261129,-1.000000,0.000996,0.739443,6232.967773,84115.307356,9.978208,1842.879452,13.751840,4267.805496,0.003392,77.785538,67542.875000
+47,6.181184,-1.000000,0.000995,0.690034,6237.704102,84051.438071,10.084672,1842.653797,13.900480,4272.115723,0.003488,77.952606,67542.875000
+48,6.220465,-1.000000,0.000995,0.706364,6233.266113,84111.281385,10.001536,1842.687614,13.786976,4267.716888,0.003872,78.241539,67542.875000
+49,6.119531,-1.000000,0.000995,0.761235,6234.041992,84100.813029,10.041984,1844.635368,13.827808,4267.366623,0.013856,77.213409,67542.875000
+50,6.152937,-1.000000,0.000995,0.731480,6234.004395,84101.320246,9.974592,1841.713345,13.778816,4269.894623,0.003584,77.820740,67542.875000
+51,6.141458,-1.000000,0.000994,0.861830,6237.763672,84050.635385,10.061952,1843.689899,13.798240,4271.625534,0.003168,77.756546,67542.875000
+52,6.195577,-1.000000,0.000994,0.969734,6232.682617,84119.155780,10.078816,1842.999317,13.830496,4267.068436,0.003520,77.862244,67542.875000
+53,6.090905,-1.000000,0.000994,0.770542,6233.434082,84109.014887,9.983456,1844.796059,13.851936,4266.444138,0.003680,77.542526,67542.875000
+54,5.978177,-1.000000,0.000994,0.803373,6234.541992,84094.068282,10.003680,1842.211903,13.823808,4269.925323,0.003392,77.590561,67542.875000
+55,6.043360,-1.000000,0.000994,0.800277,6235.530762,84080.733467,9.997920,1843.892315,13.763360,4268.063919,0.003680,79.016579,67542.875000
+56,6.009968,-1.000000,0.000993,0.623902,6232.807617,84117.468756,10.013504,1843.419098,13.748576,4266.609604,0.003456,78.140770,67542.875000
+57,6.063151,-1.000000,0.000993,0.615383,6233.230469,84111.762372,10.061088,1845.186081,13.843232,4265.690109,0.003744,77.620796,67542.875000
+58,5.994148,-1.000000,0.000993,0.623402,6236.083008,84073.287566,10.006496,1844.103100,13.791040,4269.742859,0.003648,77.556450,67542.875000
+59,6.094391,-1.000000,0.000993,0.707147,6236.016113,84074.189431,10.040128,1842.711323,13.839424,4270.132843,0.003424,78.273788,67542.875000
+60,6.054206,-1.000000,0.000992,0.775001,6234.301758,84097.308787,10.120544,1842.350121,13.877152,4268.606628,0.003520,78.411774,67542.875000
+61,6.065508,-1.000000,0.000992,0.967544,6234.533691,84094.180247,10.089792,1843.181850,13.817792,4268.884888,0.003296,77.521667,67542.875000
+62,5.917441,-1.000000,0.000992,0.584266,6239.319824,84029.672267,10.041856,1844.008507,13.793536,4273.065186,0.003328,77.396095,67542.875000
+63,6.036139,-1.000000,0.000992,0.768486,6236.318848,84070.108153,11.017984,1844.060223,13.772160,4268.692154,0.003712,77.815521,67542.875000
+64,5.989869,-1.000000,0.000991,0.892987,6232.823730,84117.251293,10.024224,1843.508774,13.784480,4265.962555,0.003456,78.691010,67542.875000
+65,5.879501,-1.000000,0.000991,0.837064,6236.106934,84072.965006,10.067968,1844.289894,13.779872,4268.698380,0.003328,78.339424,67542.875000
+66,5.985017,-1.000000,0.000991,0.650763,6235.733398,84078.001175,9.983392,1844.768772,13.734432,4268.689850,0.003968,77.834526,67542.875000
+67,5.998262,-1.000000,0.000990,0.569237,6237.871094,84049.187955,9.987040,1844.683453,13.764512,4271.132324,0.003616,77.626755,67542.875000
+68,5.877551,-1.000000,0.000990,0.613543,6238.312500,84043.240860,10.058720,1845.140160,13.807840,4270.489151,0.003488,77.818977,67542.875000
+69,5.882884,-1.000000,0.000990,0.612444,6237.937500,84048.293206,9.952576,1844.858055,13.763584,4271.095322,0.003136,77.533379,67542.875000
+70,5.904002,-1.000000,0.000989,0.621426,6235.324707,84083.512028,10.039232,1843.420071,13.799616,4270.031143,0.003584,77.190147,67542.875000
+71,5.915423,-1.000000,0.000989,0.678588,6234.292969,84097.427347,10.016000,1844.487190,13.765184,4267.175858,0.003680,78.073059,67542.875000
+72,5.838678,-1.000000,0.000989,0.772944,6237.486328,84054.372614,10.027808,1843.553535,13.727136,4270.061905,0.003328,78.467262,67542.875000
+73,5.849942,-1.000000,0.000989,0.706622,6235.878906,84076.039301,10.005312,1844.008507,13.804352,4269.854889,0.003168,77.490082,67542.875000
+74,5.814485,-1.000000,0.000988,0.615948,6242.529785,83986.463508,9.933536,1844.354988,13.720448,4276.365402,0.003488,77.467842,67542.875000
+75,5.894197,-1.000000,0.000988,0.554225,6238.767578,84037.110444,9.959936,1844.837856,13.741696,4271.949188,0.003488,77.533821,67542.875000
+76,5.706525,-1.000000,0.000988,0.728358,6237.708008,84051.385436,10.005600,1845.072029,13.757600,4270.406006,0.003488,77.724831,67542.875000

logs/csv/CP_Training_log10_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 4096
+  vocab_size: 50304  (tying=0)
+  B: 2
+  T: 4096
+  world_size: 2  (dp=2 x cp=1)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 32
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 1002
+  warmup_steps: 1
+  checkpointing: 0
+  run: 10
+  GPU Memory Used (rank 0): 27170.9 MB / 143155.9 MB

logs/csv/CP_Training_log8.csv

@@ -788,3 +788,214 @@ step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,tim
 786,2.843601,2.988807,0.001000,0.195711,5942.064941,176466.600473,7.378080,1814.773567,12.890880,4044.114380,0.003392,49.984798,95436.937500
 787,2.760746,2.988807,0.001000,0.232236,5935.569824,176659.702615,7.420448,1812.215981,12.864832,4040.333908,0.003360,49.867935,95436.937500
 788,2.770452,2.988807,0.001000,0.211091,5937.715332,176595.869179,7.362848,1813.320442,12.877088,4041.262878,0.002848,49.949760,95436.937500
+789,2.776759,2.988807,0.001000,0.209110,5955.424316,176070.745641,7.372576,1816.051895,12.904896,4056.157806,0.003200,50.016895,95436.937500
+790,2.744757,2.988807,0.001000,0.204654,5949.186035,176255.372383,7.412384,1814.410400,12.916672,4051.528305,0.003520,49.938625,95436.937500
+791,2.800298,2.988807,0.001000,0.197290,5941.709961,176477.143262,7.359904,1818.781387,12.897472,4039.760620,0.003168,49.993279,95436.937500
+792,2.741198,2.988807,0.001000,0.191320,5938.078613,176585.065353,7.339392,1813.704994,12.912864,4041.351974,0.003808,49.886593,95436.937500
+793,2.769120,2.988807,0.001000,0.201557,5949.515137,176245.622694,7.401888,1809.536285,12.880960,4056.731918,0.003648,50.029758,95436.937500
+794,2.708147,2.988807,0.001000,0.208675,5947.613770,176301.965903,7.355168,1812.419067,12.935616,4051.985779,0.003904,49.986687,95436.937500
+795,2.766753,2.988807,0.001000,0.210628,5956.694824,176033.191383,7.368992,1816.394714,12.913984,4057.199478,0.003264,49.937794,95436.937500
+796,2.777637,2.988807,0.001000,0.193652,5958.092285,175991.903082,7.361984,1815.032837,12.883040,4060.092300,0.003744,49.832481,95436.937500
+797,2.785909,2.988807,0.001000,0.186919,5953.326172,176132.798662,7.373760,1817.369980,12.895616,4052.707840,0.003744,50.095745,95436.937500
+798,2.737992,2.988807,0.001000,0.185141,5954.086426,176110.308957,7.361472,1815.411423,12.880480,4055.546112,0.003456,49.970303,95436.937500
+799,2.741153,2.988807,0.001000,0.191488,5953.370605,176131.484077,7.337696,1816.108795,12.896640,4054.253952,0.015040,49.847744,95436.937500
+800,2.787261,2.988807,0.001000,0.183715,5895.170410,177870.345901,7.404800,1798.480614,12.917312,4013.514450,0.003584,49.785183,95436.937500
+801,2.733080,2.988807,0.001000,0.183383,5924.034180,177003.705278,7.347040,1809.646278,12.903072,4031.238190,0.003520,49.983967,95436.937500
+802,2.765313,2.988807,0.001000,0.201015,5917.938965,177186.011250,7.358592,1806.513840,12.900800,4028.450180,0.003296,49.889217,95436.937500
+803,2.801133,2.988807,0.001000,0.210251,5938.800293,176563.606835,7.355616,1809.757446,12.880224,4045.912521,0.003776,50.010784,95436.937500
+804,2.702775,2.988807,0.000999,0.213116,5920.279785,177115.953646,7.516896,1810.454308,12.893120,4026.470078,0.003904,49.879360,95436.937500
+805,2.681854,2.988807,0.000999,0.214506,5954.048828,176111.421029,7.694592,1812.130989,12.936192,4057.514938,0.003168,50.020542,95436.937500
+806,2.755555,2.988807,0.000998,0.216405,5943.058105,176437.110557,7.444000,1812.263489,12.888576,4047.500473,0.003584,49.913120,95436.937500
+807,2.967290,2.988807,0.000997,0.237312,5938.364746,176576.556819,7.383808,1807.835793,12.876416,4047.439835,0.003328,49.936577,95436.937500
+808,2.861712,2.988807,0.000996,0.226572,5945.412598,176367.238232,7.312032,1809.357887,12.860736,4052.888733,0.004000,49.979584,95436.937500
+809,2.802719,2.988807,0.000996,0.237319,5957.383789,176012.833339,7.329312,1814.528801,12.883264,4059.798508,0.003328,49.932415,95436.937500
+810,2.682600,2.988807,0.000994,0.211690,5924.755371,176982.159486,7.283360,1808.387497,12.898944,4033.441116,0.003712,49.844578,95436.937500
+811,2.853862,2.988807,0.000993,0.219492,5940.732422,176506.182325,7.315104,1813.107849,12.907424,4044.638214,0.003872,49.911488,95436.937500
+812,2.667439,2.988807,0.000992,0.186758,5941.554199,176481.769726,7.349120,1812.162239,12.913536,4046.267487,0.003168,49.949886,95436.937500
+813,2.720612,2.988807,0.000991,0.198646,5953.563477,176125.778137,7.352864,1815.831444,12.903776,4054.745956,0.003552,49.865440,95436.937500
+814,2.735756,2.988807,0.000989,0.203492,5952.668457,176152.259708,7.408608,1817.387802,12.910240,4052.126190,0.003264,49.934559,95436.937500
+815,2.755608,2.988807,0.000988,0.173240,5963.339844,175837.035533,7.324576,1815.300804,12.908864,4064.847824,0.003360,49.979809,95436.937500
+816,2.777060,2.988807,0.000986,0.175868,5954.592285,176095.347890,7.346912,1814.129814,12.916608,4057.355347,0.003232,49.926785,95436.937500
+817,2.674704,2.988807,0.000984,0.179667,5946.202637,176343.805293,7.420576,1817.347878,12.986336,4045.568390,0.003328,49.877056,95436.937500
+818,2.701495,2.988807,0.000982,0.195320,5950.616211,176213.011028,7.584096,1812.008408,13.190304,4054.752350,0.003200,49.956993,95436.937500
+819,2.783086,2.988807,0.000980,0.251365,5950.958008,176202.890127,7.572416,1812.492699,13.180096,4054.620071,0.003328,50.002335,95436.937500
+820,2.724981,2.988807,0.000978,0.204131,5957.196289,176018.373261,7.550528,1816.630272,13.175008,4056.697693,0.003456,50.016735,95436.937500
+821,2.725283,2.988807,0.000976,0.231136,5960.683594,175915.393513,7.549952,1816.469894,13.171456,4058.978149,0.003200,51.387104,95436.937500
+822,2.721730,2.988807,0.000973,0.219498,5954.608887,176094.856933,7.521792,1813.541954,13.183552,4056.628448,0.003840,50.653538,95436.937500
+823,2.706247,2.988807,0.000971,0.195403,5951.255371,176194.085889,7.589568,1816.011169,13.173600,4050.861710,0.003264,50.612831,95436.937500
+824,2.788187,2.988807,0.000968,0.187277,5945.181152,176374.104191,7.549312,1820.016991,13.197664,4041.101425,0.003328,49.916065,95436.937500
+825,2.741726,2.988807,0.000966,0.183321,5900.059570,177722.951354,7.591968,1798.581268,13.180864,4017.649033,0.003168,49.915649,95436.937500
+826,2.855867,2.988807,0.000963,0.205709,5915.410156,177261.757393,7.550816,1808.542671,13.148128,4023.120285,0.003424,49.943359,95436.937500
+827,2.719985,2.988807,0.000960,0.203208,5919.104492,177151.121658,7.573824,1806.828476,13.157472,4028.554367,0.003200,49.961662,95436.937500
+828,2.723501,2.988807,0.000957,0.178775,5928.406250,176873.168906,7.669920,1808.184578,13.150944,4036.337280,0.003712,49.933632,95436.937500
+829,2.741814,2.988807,0.000954,0.170937,5931.511230,176780.580742,7.644768,1810.784737,13.180576,4036.810989,0.003840,49.913120,95436.937500
+830,2.722056,2.988807,0.000951,0.159499,5946.195801,176344.008023,7.673984,1814.265762,13.161344,4047.914658,0.003648,50.021118,95436.937500
+831,2.745772,2.988807,0.000948,0.155089,5951.373047,176190.602024,7.604160,1815.391022,13.189536,4052.103485,0.003232,49.825920,95436.937500
+832,2.765649,2.988807,0.000944,0.165793,5942.529297,176452.811188,7.560224,1812.159912,13.199904,4046.567551,0.003808,49.963806,95436.937500
+833,2.809209,2.988807,0.000941,0.183554,5955.916504,176056.195434,7.633120,1813.905273,13.175232,4058.250916,0.003168,49.940865,95436.937500
+834,2.834930,2.988807,0.000937,0.184135,5962.148926,175872.158353,8.462624,1817.217667,13.188416,4060.218384,0.003360,50.013344,95436.937500
+835,2.805997,2.988807,0.000934,0.216358,5937.666504,176597.321407,7.589472,1812.032990,13.206272,4041.829788,0.003872,49.955776,95436.937500
+836,2.751029,2.988807,0.000930,0.214813,5952.364258,176161.262077,7.559328,1816.941833,13.163936,4051.647202,0.003424,49.928062,95436.937500
+837,2.801919,2.988807,0.000926,0.205518,5950.255371,176223.697069,7.632064,1811.831886,13.185088,4054.778473,0.003232,49.818176,95436.937500
+838,2.842059,2.988807,0.000922,0.369581,5946.569336,176332.930933,7.672192,1819.212555,13.201696,4043.669830,0.003136,49.833984,95436.937500
+839,2.724386,2.988807,0.000918,0.172150,5954.804199,176089.081172,7.616384,1814.501465,13.172992,4056.425385,0.003680,50.025440,95436.937500
+840,2.785441,2.988807,0.000914,0.156380,5950.187988,176225.692712,7.928576,1817.371742,13.182816,4048.661743,0.003712,50.017281,95436.937500
+841,2.743148,2.988807,0.000910,0.175145,5946.797363,176326.169524,7.553888,1814.643776,13.181152,4048.262680,0.003808,50.074402,95436.937500
+842,2.741702,2.988807,0.000906,0.177996,5948.441895,176277.421650,7.585664,1816.967392,13.209536,4047.541138,0.003840,50.011906,95436.937500
+843,2.755732,2.988807,0.000901,0.183384,5945.697266,176358.794125,7.686304,1815.443192,13.182848,4046.444351,0.003296,49.836414,95436.937500
+844,2.752215,2.988807,0.000897,0.171924,5952.953613,176143.821726,7.661536,1814.610527,13.174336,4054.452026,0.003520,50.016544,95436.937500
+845,2.820490,2.988807,0.000892,0.180827,5947.114258,176316.773908,7.603424,1815.210640,13.164064,4046.856201,0.003776,50.549824,95436.937500
+846,2.790235,2.988807,0.000888,0.171260,5958.750977,175972.448610,7.670592,1816.109940,13.174432,4058.765656,0.003264,50.001759,95436.937500
+847,2.829271,2.988807,0.000883,0.167824,5949.566406,176244.103923,7.533472,1813.684532,13.179360,4052.033997,0.003168,50.027969,95436.937500
+848,2.834256,2.988807,0.000878,0.187648,5962.573242,175859.642709,7.661472,1817.472496,13.178336,4061.035400,0.003840,50.084450,95436.937500
+849,2.803730,2.988807,0.000873,0.155694,5960.046387,175934.201173,7.548032,1815.790085,13.181312,4060.162704,0.003776,50.199009,95436.937500
+850,2.739548,2.988807,0.000868,0.143233,5900.280273,177716.303532,7.571200,1800.816689,13.132832,4015.721756,0.003392,49.879009,95436.937500
+851,2.712695,2.988807,0.000863,0.168893,5927.515625,176899.744570,7.473440,1810.220383,13.178976,4033.569092,0.003776,49.964993,95436.937500
+852,2.741832,2.988807,0.000858,0.178976,5933.818848,176711.832114,7.779104,1809.332100,13.184096,4040.464355,0.003744,49.935966,95436.937500
+853,2.713052,2.988807,0.000853,0.170074,5929.506836,176840.339174,7.549696,1812.381798,13.167872,4033.404617,0.003808,49.835678,95436.937500
+854,2.742614,2.988807,0.000848,0.162478,5939.276855,176549.439522,7.609088,1814.485954,13.163520,4040.863480,0.003904,49.924671,95436.937500
+855,2.781667,2.988807,0.000842,0.163987,5941.976074,176469.239678,7.653600,1817.639679,13.191424,4040.198029,0.003552,50.037376,95436.937500
+856,2.709754,2.988807,0.000837,0.170186,5939.186035,176552.139265,7.619904,1812.359276,13.171136,4042.669693,0.003488,50.310944,95436.937500
+857,2.686680,2.988807,0.000831,0.174913,5942.864746,176442.851184,10.692288,1810.433701,13.193792,4045.589813,0.003904,49.842529,95436.937500
+858,2.785247,2.988807,0.000826,0.190831,5944.686523,176388.779436,7.581696,1812.421730,13.188768,4048.529556,0.003680,49.876831,95436.937500
+859,2.766020,2.988807,0.000820,0.159741,5946.408691,176337.694635,7.600800,1813.367676,13.185504,4049.252808,0.003872,49.951839,95436.937500
+860,2.677573,2.988807,0.000815,0.173482,5949.049805,176259.408549,7.530688,1810.764343,13.178976,4054.442642,0.003712,49.966400,95436.937500
+861,2.716818,2.988807,0.000809,0.170767,5949.045898,176259.524284,7.567776,1816.117607,13.171776,4049.237045,0.003712,49.856670,95436.937500
+862,2.786133,2.988807,0.000803,0.157780,5953.006348,176142.261366,7.608000,1815.968178,13.149152,4053.225479,0.003360,50.018784,95436.937500
+863,2.790838,2.988807,0.000797,0.147667,5954.120117,176109.312436,7.557504,1820.810364,13.198592,4049.281174,0.003328,50.162434,95436.937500
+864,2.752659,2.988807,0.000791,0.147443,5960.398926,175923.795212,7.577568,1816.157852,13.148736,4060.505066,0.003168,49.858112,95436.937500
+865,2.717798,2.988807,0.000785,0.144325,5956.028809,176052.875783,7.627552,1816.652817,13.239392,4055.396805,0.003872,50.002720,95436.937500
+866,2.714525,2.988807,0.000779,0.148175,5956.256348,176046.150266,7.601376,1815.089600,13.198624,4057.308243,0.003680,49.865376,95436.937500
+867,2.828845,2.988807,0.000773,0.150143,5963.384277,175835.725359,7.578816,1818.279724,13.200288,4061.304459,0.003296,49.847519,95436.937500
+868,2.704845,2.988807,0.000767,0.140785,5948.657227,176271.040684,7.604000,1814.538246,13.244224,4049.980377,0.003776,49.957314,95436.937500
+869,2.721409,2.988807,0.000761,0.137573,5952.241211,176164.903746,7.660160,1815.065155,13.164640,4053.456329,0.003744,49.870625,95436.937500
+870,2.608265,2.988807,0.000754,0.135357,5964.684570,175797.393414,7.489728,1816.964394,13.170624,4064.034164,0.003712,49.893440,95436.937500
+871,2.698054,2.988807,0.000748,0.160469,5947.617188,176301.864586,7.608128,1815.984550,13.171584,4047.784912,0.003200,50.030945,95436.937500
+872,2.700965,2.988807,0.000742,0.149692,5964.659180,175798.141757,7.679712,1816.702049,13.163776,4064.107208,0.003552,49.917538,95436.937500
+873,2.816442,2.988807,0.000735,0.147774,5957.333984,176014.304847,7.597216,1814.503494,13.191008,4058.954895,0.003936,50.013855,95436.937500
+874,2.724926,2.988807,0.000729,0.150276,5963.894043,175820.695748,7.567136,1817.545967,13.194752,4061.682709,0.003584,50.846687,95436.937500
+875,2.726554,2.988807,0.000722,0.129432,5888.628906,178067.936814,7.441984,1795.976189,13.127424,4009.153397,0.003168,49.853439,95436.937500
+876,2.638878,2.988807,0.000716,0.129349,5913.936035,177305.942061,7.591872,1809.286118,13.109056,4021.034637,0.003936,49.923073,95436.937500
+877,2.694532,2.988807,0.000709,0.132979,5932.075195,176763.774139,7.567712,1807.443230,13.146400,4040.922485,0.003168,49.994816,95436.937500
+878,2.668688,2.988807,0.000702,0.138398,5929.321289,176845.873057,7.535712,1805.443779,13.136032,4040.231567,0.003520,49.992962,95436.937500
+879,2.698247,2.988807,0.000696,0.141659,5937.312500,176607.850774,7.607616,1813.088547,13.185024,4040.520340,0.003680,49.913246,95436.937500
+880,2.724090,2.988807,0.000689,0.146695,5940.464844,176514.132746,7.696000,1808.564285,13.197920,4047.669922,0.003776,50.031231,95436.937500
+881,2.647027,2.988807,0.000682,0.132685,5953.248535,176135.095622,7.603424,1811.902412,13.174112,4057.500916,0.003872,49.973663,95436.937500
+882,2.741259,2.988807,0.000676,0.138875,5945.207520,176373.321967,7.677376,1817.101768,13.188960,4044.202499,0.003200,50.030464,95436.937500
+883,2.676762,2.988807,0.000669,0.135611,5948.227539,176283.774135,7.592384,1816.526718,13.180416,4047.908432,0.003424,50.031456,95436.937500
+884,2.662589,2.988807,0.000662,0.131130,5950.149414,176226.835165,7.592224,1813.823486,13.166048,4052.470673,0.003296,50.054722,95436.937500
+885,2.716739,2.988807,0.000655,0.124578,5945.113281,176376.117728,7.558752,1813.130623,13.156608,4047.985748,0.003584,50.208672,95436.937500
+886,2.650722,2.988807,0.000648,0.122145,5941.006836,176498.029536,7.546368,1813.805473,13.146176,4043.363022,0.003360,50.061184,95436.937500
+887,2.681291,2.988807,0.000641,0.119859,5957.929688,175996.706071,7.611712,1819.360207,13.204544,4054.035645,0.003456,50.611969,95436.937500
+888,2.679257,2.988807,0.000634,0.110791,5956.293457,176045.053449,7.568608,1815.685890,13.176000,4056.610245,0.003648,49.948479,95436.937500
+889,2.730773,2.988807,0.000627,0.115457,5941.537109,176482.277346,7.558784,1808.598724,13.146240,4048.538712,0.003200,50.728928,95436.937500
+890,2.747416,2.988807,0.000620,0.113413,5949.668945,176241.066459,7.594432,1813.903557,13.162656,4052.123184,0.003776,49.871361,95436.937500
+891,2.767974,2.988807,0.000613,0.116666,5946.815918,176325.619367,7.580576,1811.603317,13.176992,4051.341431,0.003904,50.073952,95436.937500
+892,2.712966,2.988807,0.000606,0.128414,5947.699707,176299.418540,7.601952,1815.245277,13.168512,4048.671646,0.003712,50.032223,95436.937500
+893,2.688213,2.988807,0.000599,0.137009,5958.311035,175985.441816,7.625824,1814.352188,13.191840,4060.133667,0.003840,50.025055,95436.937500
+894,2.678957,2.988807,0.000592,0.136385,5962.106445,175873.411456,7.610016,1814.748352,13.163840,4059.811401,0.003296,53.739231,95436.937500
+895,2.652299,2.988807,0.000585,0.119470,5947.447266,176306.901628,7.588960,1813.140823,13.179744,4050.520081,0.003360,49.916351,95436.937500
+896,2.647521,2.988807,0.000578,0.110283,5961.592773,175888.565330,7.510976,1816.531868,13.169056,4061.214844,0.003616,50.057312,95436.937500
+897,2.694245,2.988807,0.000571,0.116113,5956.573730,176036.770037,7.529568,1814.793121,13.168448,4058.050339,0.003552,49.895329,95436.937500
+898,2.683159,2.988807,0.000564,0.107818,5953.708008,176121.502537,7.661088,1815.199104,13.203040,4054.707352,0.003744,49.972065,95436.937500
+899,2.668508,2.988807,0.000557,0.105812,5955.169434,176078.281515,7.601280,1817.312988,13.203392,4053.013092,0.003584,49.969921,95436.937500
+900,2.664789,2.988807,0.000550,0.105697,5882.530273,178252.546312,7.554656,1795.487297,13.171744,4003.379379,0.003680,49.861599,95436.937500
+901,2.650917,2.988807,0.000543,0.101536,5929.829590,176830.713954,7.570432,1810.629936,13.148480,4035.458542,0.003200,49.973408,95436.937500
+902,2.640871,2.988807,0.000536,0.102694,5929.495605,176840.674109,7.511392,1810.309410,13.150496,4035.501068,0.003200,49.931137,95436.937500
+903,2.666461,2.988807,0.000529,0.100467,5925.335938,176964.818714,7.554592,1807.639381,13.153792,4033.988525,0.003328,49.870975,95436.937500
+904,2.598424,2.988807,0.000522,0.099237,5937.469238,176603.188651,7.588864,1813.569824,13.140576,4040.145065,0.003360,49.981377,95436.937500
+905,2.664531,2.988807,0.000515,0.102724,5949.605957,176242.932317,7.729440,1810.500572,13.227808,4054.504318,0.003232,50.086910,95436.937500
+906,2.598361,2.988807,0.000508,0.099630,5948.835449,176265.759736,7.634432,1813.348396,13.212128,4051.419693,0.003744,50.005375,95436.937500
+907,2.670407,2.988807,0.000501,0.099800,5953.570312,176125.575908,7.654784,1812.969727,13.161536,4056.725540,0.003872,49.971329,95436.937500
+908,2.615990,2.988807,0.000494,0.105959,5959.630371,175946.482367,7.536576,1814.778748,13.179072,4061.083191,0.003488,49.987518,95436.937500
+909,2.686617,2.988807,0.000487,0.098641,5946.640625,176330.817032,7.565792,1815.422523,13.194400,4047.054687,0.003840,50.168640,95436.937500
+910,2.685745,2.988807,0.000480,0.093319,5953.147949,176138.071646,7.555264,1817.701523,13.189376,4051.596588,0.003776,49.944576,95436.937500
+911,2.627549,2.988807,0.000473,0.094446,5960.513184,175920.422907,7.615072,1814.622986,13.184128,4061.953537,0.003360,50.118401,95436.937500
+912,2.653412,2.988807,0.000466,0.095126,5946.169922,176344.775507,7.602624,1812.746460,13.187520,4049.106949,0.003744,50.024158,95436.937500
+913,2.612473,2.988807,0.000459,0.091752,5949.235352,176253.911307,7.540320,1814.930832,13.133952,4050.645309,0.003296,49.956703,95436.937500
+914,2.612973,2.988807,0.000452,0.093348,5956.102539,176050.696428,7.498944,1818.688683,12.915200,4053.742905,0.003328,50.078049,95436.937500
+915,2.619504,2.988807,0.000445,0.098125,5951.572754,176184.689889,7.413312,1817.405563,12.935072,4050.778564,0.003424,49.927170,95436.937500
+916,2.639683,2.988807,0.000438,0.094372,5951.535156,176185.802901,7.507584,1815.492737,12.932032,4052.472000,0.003296,50.081280,95436.937500
+917,2.680348,2.988807,0.000431,0.093898,5956.324707,176044.129824,7.476448,1814.851891,12.946272,4057.940887,0.003328,49.980576,95436.937500
+918,2.669458,2.988807,0.000424,0.097843,5951.980957,176172.606662,7.370048,1815.433830,12.934112,4053.059555,0.003488,50.027809,95436.937500
+919,2.701620,2.988807,0.000418,0.100182,5955.540527,176067.309959,7.441472,1814.157150,12.935168,4057.994308,0.003328,49.968895,95436.937500
+920,2.726842,2.988807,0.000411,0.095452,5950.714844,176210.090306,7.414176,1816.019005,12.953248,4051.222229,0.003680,50.059486,95436.937500
+921,2.703319,2.988807,0.000404,0.103068,5952.192383,176166.348895,7.468960,1814.950012,12.935488,4053.801575,0.003904,49.971134,95436.937500
+922,2.682874,2.988807,0.000398,0.093735,5959.246582,175957.813721,7.444416,1819.175591,12.918656,4056.591446,0.003904,50.002434,95436.937500
+923,2.661068,2.988807,0.000391,0.094935,5950.914551,176204.176863,7.388352,1821.311783,12.942656,4046.238571,0.003168,50.030399,95436.937500
+924,2.570555,2.988807,0.000384,0.095194,5944.919922,176381.854387,7.380672,1815.130646,12.928800,4046.075333,0.003424,50.133888,95436.937500
+925,2.700989,2.988807,0.000378,0.092634,5963.465332,175833.335421,7.332160,1797.240036,12.978656,4018.596649,0.003456,49.887585,95436.937500
+926,2.650601,2.988807,0.000371,0.092948,5917.538086,177198.014575,7.350816,1804.788383,13.032384,4029.405365,0.003936,49.930401,95436.937500
+927,2.638551,2.988807,0.000365,0.090208,5919.453125,177140.688144,7.371424,1810.334854,12.910080,4025.794815,0.003328,49.932926,95436.937500
+928,2.647935,2.988807,0.000358,0.093268,5933.505859,176721.153539,7.646400,1811.489799,12.916896,4038.481125,0.003392,49.910816,95436.937500
+929,2.729273,2.988807,0.000352,0.089797,5933.367188,176725.283783,7.566784,1810.317177,13.000640,4039.283752,0.003072,50.016449,95436.937500
+930,2.663484,2.988807,0.000346,0.107330,5938.493164,176572.738409,7.482432,1812.084984,12.925248,4042.706696,0.003168,50.043262,95436.937500
+931,2.610590,2.988807,0.000339,0.094930,5939.702148,176536.798276,7.457856,1815.988670,12.912128,4040.292969,0.003424,49.936352,95436.937500
+932,2.578015,2.988807,0.000333,0.092170,5933.518066,176720.789971,7.446240,1813.590027,12.916992,4036.673630,0.003680,49.871201,95436.937500
+933,2.631655,2.988807,0.000327,0.096826,5947.120605,176316.585716,7.469152,1816.650246,12.922176,4047.030334,0.003616,49.999969,95436.937500
+934,2.691433,2.988807,0.000321,0.090589,5952.823730,176147.664953,7.418464,1814.374611,12.919456,4055.176376,0.003616,49.936321,95436.937500
+935,2.621213,2.988807,0.000315,0.091107,5955.136719,176079.248810,7.460320,1813.408028,12.989632,4058.340424,0.003648,49.950336,95436.937500
+936,2.640804,2.988807,0.000309,0.093354,5947.369629,176309.203131,7.383200,1813.680130,12.918528,4050.482117,0.003552,49.975582,95436.937500
+937,2.636666,2.988807,0.000303,0.091758,5953.366699,176131.599644,7.447008,1817.258560,12.926880,4052.826614,0.003840,49.906433,95436.937500
+938,2.627810,2.988807,0.000297,0.089993,5955.289551,176074.730046,7.426528,1813.231079,12.924288,4058.699402,0.003200,50.069664,95436.937500
+939,2.699911,2.988807,0.000291,0.096143,5943.609863,176420.731528,7.382624,1814.034561,13.032256,4046.141937,0.003200,49.930401,95436.937500
+940,2.640586,2.988807,0.000285,0.091082,5956.481445,176039.497416,7.372736,1819.047325,12.927872,4054.213440,0.003168,49.864513,95436.937500
+941,2.598037,2.988807,0.000280,0.086820,5956.917969,176026.597227,7.431040,1812.545258,12.922240,4061.086136,0.003904,49.969601,95436.937500
+942,2.670021,2.988807,0.000274,0.087945,5958.246582,175987.345533,7.407520,1815.646988,12.909376,4059.086304,0.003584,50.016254,95436.937500
+943,2.566868,2.988807,0.000269,0.091339,5954.662109,176093.283001,7.425696,1814.401947,12.919712,4056.935760,0.003264,49.947647,95436.937500
+944,2.649738,2.988807,0.000263,0.097048,5959.548340,175948.904213,7.417568,1820.872643,12.975552,4055.254730,0.003552,49.976158,95436.937500
+945,2.594111,2.988807,0.000258,0.090864,5961.929688,175878.625707,7.365312,1813.853661,12.917056,4064.889984,0.003072,49.863937,95436.937500
+946,2.605718,2.988807,0.000252,0.145210,5959.437500,175952.176695,7.404704,1813.662262,12.917600,4062.535950,0.003776,49.911457,95436.937500
+947,2.552562,2.988807,0.000247,0.089215,5957.179688,176018.863792,7.358624,1809.863419,12.945952,4063.757919,0.003168,49.966881,95436.937500
+948,2.608181,2.988807,0.000242,0.087132,5953.166504,176137.522663,7.367584,1818.572556,12.934048,4051.292221,0.003712,49.953056,95436.937500
+949,2.624990,2.988807,0.000237,0.089756,5956.418945,176041.344578,7.426720,1817.688225,12.955712,4055.352280,0.003264,49.973248,95436.937500
+950,2.645914,2.988807,0.000232,0.092899,5904.699707,177583.289926,7.446272,1800.588776,12.879456,4020.765869,0.012576,49.963295,95436.937500
+951,2.558727,2.988807,0.000227,0.084885,5936.286621,176638.371246,7.331968,1807.978310,12.920032,4045.104309,0.003456,49.862751,95436.937500
+952,2.652134,2.988807,0.000222,0.102298,5935.326172,176666.954711,10.464544,1810.428452,12.896736,4038.495514,0.003648,50.048031,95436.937500
+953,2.672426,2.988807,0.000217,0.095819,5936.773438,176623.886870,7.438272,1811.216194,12.913472,4042.304077,0.003008,49.917377,95436.937500
+954,2.657829,2.988807,0.000212,0.094092,5947.320801,176310.650648,7.451328,1810.662918,12.919456,4053.222214,0.003712,50.054337,95436.937500
+955,2.892116,2.988807,0.000208,0.183617,5939.088379,176555.042307,7.471552,1812.018181,12.929536,4043.566620,0.003392,50.014561,95436.937500
+956,2.542175,2.988807,0.000203,0.098078,5940.335449,176517.977640,7.562976,1813.459908,12.924192,4043.248886,0.003840,49.937183,95436.937500
+957,2.618965,2.988807,0.000199,0.090572,5958.593750,175977.091910,7.398304,1815.165977,12.927968,4060.100952,0.003232,49.994686,95436.937500
+958,2.630756,2.988807,0.000194,0.089107,5946.620605,176331.410656,7.438240,1811.865707,12.931392,4051.400085,0.003840,49.981182,95436.937500
+959,2.612083,2.988807,0.000190,0.086179,5947.681152,176299.968533,7.441472,1813.713921,12.929952,4050.569046,0.003424,50.014080,95436.937500
+960,2.655385,2.988807,0.000186,0.087518,5939.148438,176553.256925,7.475424,1811.993958,12.905536,4043.609421,0.003776,50.135296,95436.937500
+961,2.651754,2.988807,0.000182,0.098374,5957.658203,176004.726060,7.431808,1815.005051,12.905248,4059.298462,0.003232,49.957985,95436.937500
+962,2.620201,2.988807,0.000178,0.086986,5952.988281,176142.795930,7.431296,1816.391411,12.919360,4053.270218,0.003840,49.917278,95436.937500
+963,2.628344,2.988807,0.000174,0.086640,5939.736816,176535.767899,7.423776,1816.735519,12.929088,4039.826050,0.003264,49.827457,95436.937500
+964,2.542377,2.988807,0.000170,0.088012,5944.774414,176386.171613,7.475712,1812.849503,12.909856,4048.525314,0.003424,49.958912,95436.937500
+965,2.614022,2.988807,0.000166,0.088807,5959.780273,175942.056903,7.516960,1814.566505,12.902592,4061.842239,0.003552,49.894562,95436.937500
+966,2.584664,2.988807,0.000163,0.085318,5946.958984,176321.377490,7.458752,1816.654297,12.947040,4046.916718,0.003584,49.916672,95436.937500
+967,2.542435,2.988807,0.000159,0.082694,5960.512207,175920.451729,7.467712,1815.616219,12.934784,4061.467407,0.003296,49.976257,95436.937500
+968,2.577236,2.988807,0.000156,0.086095,5948.880371,176264.428697,7.494304,1816.108719,12.936384,4049.406738,0.011296,49.973759,95436.937500
+969,2.535849,2.988807,0.000152,0.083083,5953.608398,176124.449212,7.418560,1814.412628,12.900352,4055.974228,0.003104,49.915615,95436.937500
+970,2.614408,2.988807,0.000149,0.087376,5947.176758,176314.920962,7.419904,1815.425301,12.919904,4048.146103,0.003360,50.126080,95436.937500
+971,2.638258,2.988807,0.000146,0.083726,5967.204590,175723.152141,7.385440,1817.584152,12.919008,4066.365295,0.003168,49.989922,95436.937500
+972,2.529380,2.988807,0.000143,0.081337,5941.315918,176488.847669,7.435232,1813.740852,12.914528,4044.256393,0.003168,49.958847,95436.937500
+973,2.657314,2.988807,0.000140,0.088359,5965.054688,175786.485612,7.410304,1816.223022,12.916640,4065.546585,0.003168,49.963009,95436.937500
+974,2.745756,2.988807,0.000137,0.100399,5947.381348,176308.855731,7.402400,1818.532410,12.910208,4045.712952,0.003296,49.863361,95436.937500
+975,2.788818,2.988807,0.000134,0.104115,5906.055664,177542.519008,7.223328,1803.263412,12.950688,4019.687943,0.003168,49.983295,95436.937500
+976,2.660657,2.988807,0.000132,0.092523,5921.580566,177077.046954,7.362400,1808.015465,12.895680,4029.677582,0.003104,50.875393,95436.937500
+977,2.754989,2.988807,0.000129,0.097973,5926.214355,176938.587959,7.286272,1809.698662,12.885184,4032.704636,0.003776,50.832031,95436.937500
+978,2.782961,2.988807,0.000127,0.088222,5938.604004,176569.442803,7.334048,1812.399452,12.917152,4042.132523,0.003808,51.035393,95436.937500
+979,2.649026,2.988807,0.000124,0.087721,5934.151855,176701.915546,7.345152,1814.270561,12.880384,4036.080109,0.003264,50.778400,95436.937500
+980,2.794215,2.988807,0.000122,0.096227,5943.669922,176418.948862,7.346560,1813.820122,12.901184,4046.618256,0.003584,50.088673,95436.937500
+981,2.708078,2.988807,0.000120,0.088402,5939.277344,176549.425008,7.386432,1813.903336,12.916160,4041.463242,0.003680,50.674335,95436.937500
+982,2.647987,2.988807,0.000118,0.091115,5947.670898,176300.272477,7.295872,1809.545029,12.890624,4054.400742,0.003424,50.843105,95436.937500
+983,2.673492,2.988807,0.000116,0.089014,5949.329102,176251.133884,7.329696,1812.455421,12.869440,4052.819870,0.003424,51.096642,95436.937500
+984,2.737326,2.988807,0.000114,0.088743,5951.180664,176196.297708,7.346048,1817.718285,12.875776,4050.519836,0.003840,49.965439,95436.937500
+985,2.697475,2.988807,0.000112,0.086123,5952.461426,176158.386421,7.296128,1814.374466,12.942848,4055.170013,0.003392,49.881504,95436.937500
+986,2.673871,2.988807,0.000111,0.085972,5947.864746,176294.526652,7.344704,1816.883186,12.907616,4047.948868,0.003648,49.959553,95436.937500
+987,2.765230,2.988807,0.000109,0.084560,5946.695312,176329.195443,7.307328,1817.796249,12.898240,4045.972946,0.003264,49.941826,95436.937500
+988,2.672553,2.988807,0.000108,0.086184,5945.650391,176360.184523,7.359264,1813.194336,12.910912,4049.504425,0.003424,49.901600,95436.937500
+989,2.674335,2.988807,0.000107,0.084857,5947.734863,176298.376458,7.347584,1814.251808,12.894976,4050.522537,0.003424,49.933216,95436.937500
+990,2.692748,2.988807,0.000106,0.086729,5951.437988,176188.679453,7.367744,1807.215714,12.917824,4061.216629,0.003680,49.911938,95436.937500
+991,2.712992,2.988807,0.000104,0.083877,5953.306641,176133.376508,7.356864,1816.397789,12.908096,4053.866959,0.003232,50.020737,95436.937500
+992,2.743290,2.988807,0.000104,0.085711,5957.268555,176016.238042,7.382016,1816.360962,12.903712,4057.984711,0.003744,49.845921,95436.937500
+993,2.844353,2.988807,0.000103,0.100867,5949.885254,176234.659200,7.385088,1816.931068,12.930496,4049.953720,0.003840,49.947552,95436.937500
+994,2.689448,2.988807,0.000102,0.082345,5939.126953,176553.895594,7.347808,1809.081841,12.926400,4047.046768,0.003872,49.912670,95436.937500
+995,2.710656,2.988807,0.000101,0.085877,5951.061035,176199.839626,7.267744,1815.058861,12.904992,4053.099670,0.003168,49.920063,95436.937500
+996,2.718275,2.988807,0.000101,0.089110,5950.007812,176231.029108,7.344480,1818.051735,12.906656,4048.508560,0.003328,50.405632,95436.937500
+997,2.700864,2.988807,0.000100,0.113435,5952.789551,176148.676357,7.286944,1815.807899,12.889952,4054.113144,0.003392,49.945343,95436.937500
+998,2.747742,2.988807,0.000100,0.085683,5945.287109,176370.960848,7.297376,1813.916756,12.912448,4047.874374,0.003232,50.538143,95436.937500
+999,2.662259,2.817160,0.000100,0.091953,8284.758789,126566.871372,7.326592,1815.829697,12.911648,4065.655655,0.003776,49.993023,95436.937500

logs/csv/CP_Training_log9.csv

@@ -0,0 +1,3 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,11.091398,-1.000000,0.001000,19.279182,8030.833008,130568.771506,8.701376,3881.846298,16.530784,4023.512527,0.003360,63.372448,94444.937500
+1,9.587653,-1.000000,0.001000,40.352005,7238.025391,144870.450628,5.636224,3138.602791,12.841120,4018.376205,0.003264,49.770977,95660.937500

logs/csv/CP_Training_log9_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 4096
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 4096
+  world_size: 4  (dp=4 x cp=1)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 2
+  warmup_steps: 1
+  checkpointing: 0
+  run: 9
+  GPU Memory Used (rank 0): 20906.9 MB / 143155.9 MB