Upload folder using huggingface_hub

logs/in2_train_csv/CP_Training_log1.csv

@@ -0,0 +1,13 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.771908,-1.000000,0.000030,0.565346,3140.228271,10434.910193,15.315136,1801.972820,7.820672,1152.986122,0.003616,135.850311,64028.875000
+1,2.735266,-1.000000,0.000060,0.442705,1404.127686,23336.908984,2.138112,665.447750,1.975936,605.859802,0.003872,122.795555,64636.875000
+2,2.923653,-1.000000,0.000090,0.460064,1634.058350,20053.139478,2.626688,879.502031,1.982304,620.304962,0.003296,123.515968,64636.875000
+3,2.522028,-1.000000,0.000120,0.422324,961.744995,34071.401636,2.201664,244.461220,1.980864,584.610565,0.003584,122.578430,64636.875000
+4,2.939733,-1.000000,0.000150,0.614175,962.684021,34038.167545,2.183712,244.372766,1.955712,585.666237,0.003712,122.575714,64636.875000
+5,2.728506,-1.000000,0.000180,0.660212,963.226807,34018.986779,2.155680,244.660545,1.954080,586.378593,0.003168,122.255295,64636.875000
+6,2.777696,-1.000000,0.000210,0.489034,963.257263,34017.911157,2.186976,244.545952,1.961760,585.298500,0.003264,123.393600,64636.875000
+7,2.714472,-1.000000,0.000240,0.540897,966.425964,33906.373803,2.195200,245.005182,1.961152,588.347595,0.003424,122.852097,64636.875000
+8,2.737436,-1.000000,0.000270,0.496506,966.042664,33919.826976,2.226272,245.006338,1.959456,588.359909,0.003232,122.519363,64636.875000
+9,2.953558,-1.000000,0.000300,0.516820,964.353516,33979.240464,2.240896,244.847872,1.966176,586.751518,0.003200,122.581726,64636.875000
+10,2.481278,-1.000000,0.000300,0.628861,964.450378,33975.827822,2.207872,244.690178,1.951424,586.476044,0.003712,123.261986,64636.875000
+11,2.391085,-1.000000,0.000200,0.530467,965.237671,33948.115566,2.198368,244.774498,1.978144,586.277565,0.003200,122.690079,64636.875000

logs/in2_train_csv/CP_Training_log10.csv

@@ -0,0 +1,27 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,1.339281,1.294896,0.000003,0.460841,15974.457031,32820.395646,3.230272,3831.259598,6.512224,4988.204834,0.003872,62.277184,107212.937500
+1,1.293751,1.294896,0.000006,0.405247,8800.057617,59577.791738,4.380864,4184.290054,6.480928,4547.717377,0.003200,48.965664,108908.937500
+2,1.312084,1.294896,0.000009,0.654968,8367.143555,62660.332833,5.121024,3780.039200,6.470624,4517.891113,0.003200,49.263969,108908.937500
+3,1.350616,1.294896,0.000012,0.406956,6207.470703,84460.809414,3.523424,1990.446243,6.477760,4149.764282,0.003520,49.138306,108908.937500
+4,1.333423,1.294896,0.000015,0.528120,6229.354980,84164.091089,4.100768,1996.354599,6.513696,4164.996765,0.003904,49.173248,108908.937500
+5,1.408367,1.294896,0.000018,0.405359,6242.764648,83983.303797,3.640768,1998.599701,6.485120,4176.157654,0.003264,49.690079,108908.937500
+6,1.340171,1.294896,0.000021,0.428254,6317.099121,82995.056742,3.656000,2035.197586,6.500960,4214.110352,0.003392,49.460510,108908.937500
+7,1.303951,1.294896,0.000024,0.487018,6402.280762,81890.816650,3.498656,2089.252090,6.496864,4245.549622,0.003360,49.250786,108908.937500
+8,1.294623,1.294896,0.000027,0.396574,6387.675781,82078.054359,3.571040,2030.895142,6.570912,4289.417786,0.003168,48.988800,108908.937500
+9,1.339558,1.294896,0.000030,0.399714,6347.899902,82592.354647,3.770304,2028.917252,6.531488,4250.892090,0.003456,49.192738,108908.937500
+10,1.296980,1.294896,0.000033,0.512690,6350.575684,82557.554798,3.533728,2022.723068,6.522592,4260.554871,0.003360,49.066944,108908.937500
+11,1.311951,1.294896,0.000036,0.439680,6326.203125,82875.619015,3.609280,2017.244186,6.510080,4240.678772,0.003840,49.934528,108908.937500
+12,1.311354,1.294896,0.000039,0.559122,6344.461914,82637.110460,3.475744,2027.788773,6.499200,4249.298279,0.003456,49.208256,108908.937500
+13,1.326638,1.294896,0.000042,0.455803,6317.663574,82987.641529,3.631520,2015.776199,6.509664,4233.496216,0.003264,50.149536,108908.937500
+14,1.368466,1.294896,0.000045,0.421233,6301.248047,83203.834558,3.639680,2009.533081,6.725248,4223.833435,0.003328,49.297855,108909.125000
+15,1.356867,1.294896,0.000048,0.613763,6520.707520,80403.544927,3.538080,2087.908768,6.450400,4361.749146,0.003360,51.546658,110860.687500
+16,1.342574,1.294896,0.000051,0.567213,6298.546387,83239.523504,3.638496,2016.333511,6.480832,4214.827820,0.003392,49.043808,111894.687500
+17,1.280889,1.294896,0.000054,0.349035,6284.050781,83431.534571,3.578976,2007.466309,6.529760,4208.839600,0.003456,49.383202,112758.687500
+18,1.320351,1.294896,0.000057,0.851928,6282.781250,83448.393178,3.693984,2008.129181,6.483776,4206.946228,0.003552,49.257759,113622.687500
+19,1.339405,1.294896,0.000060,0.439494,6309.713867,83092.198955,3.584928,2032.398819,6.503456,4209.475708,0.003328,49.477886,114518.687500
+20,1.317941,1.294896,0.000063,0.495442,6277.857910,83513.836647,3.561696,2005.930069,6.504288,4204.565796,0.003328,49.078079,115382.687500
+21,1.395253,1.294896,0.000066,0.413536,6332.917969,82787.745331,3.566784,2036.316101,6.506528,4228.003723,0.003520,50.223263,116054.687500
+22,1.330439,1.294896,0.000069,0.421238,8242.111328,63610.885503,5.593344,2568.815063,6.411456,5590.474487,0.003680,49.093441,142288.687500
+23,1.333955,1.294896,0.000072,0.535151,6246.558105,83932.301781,3.542464,2001.504044,6.499296,4177.465576,0.003488,49.374176,108909.000000
+24,1.305244,1.294896,0.000075,0.473453,6267.697266,83649.222000,3.642592,2002.647873,6.482720,4197.135193,0.003840,49.635391,108909.000000
+25,1.386446,1.294896,0.000078,0.479760,6253.282227,83842.049824,3.581952,2002.195984,6.488256,4183.623901,0.003264,49.231457,108909.000000

logs/in2_train_csv/CP_Training_log10_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 100
+  run: 10
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log11_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 100
+  run: 11
+  GPU Memory Used (rank 0): 129812.7 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log12.csv

@@ -0,0 +1,401 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,1.339281,1.294896,0.000003,0.460883,15903.497070,32966.837274,3.258208,4207.280457,6.504192,4591.857849,0.003712,62.649120,107212.937500
+1,1.293756,1.294896,0.000006,0.421186,8306.622070,63116.871764,3.587904,4053.086609,6.455520,4185.799316,0.003840,49.638847,108908.937500
+2,1.312149,1.294896,0.000009,0.569313,8234.271484,63671.449380,5.002208,4017.712753,6.442528,4147.167206,0.003520,49.853760,108908.937500
+3,1.350716,1.294896,0.000012,0.530472,6207.299805,84463.134776,3.504736,1993.893173,6.463712,4146.050659,0.003808,49.414658,108908.937500
+4,1.333467,1.294896,0.000015,0.507369,6218.856445,84306.175036,3.376608,1994.607544,6.483584,4157.083435,0.003648,49.327583,108908.937500
+5,1.408316,1.294896,0.000018,0.488558,6243.314453,83975.907979,3.548672,2002.311493,6.463136,4172.814056,0.003232,50.116001,108908.937500
+6,1.340124,1.294896,0.000021,0.549103,6298.953613,83234.142079,3.553440,2017.879608,6.511744,4212.890198,0.003648,50.034592,108908.937500
+7,1.304078,1.294896,0.000024,0.595246,6327.498535,82858.652134,3.487040,2018.346786,6.492704,4241.732483,0.003744,49.402977,108908.937500
+8,1.294508,1.294896,0.000027,0.582293,6396.759277,81961.502265,3.503552,2054.861389,6.522624,4274.229858,0.003328,49.649857,108908.937500
+9,1.339643,1.294896,0.000030,0.557250,6400.497559,81913.631745,3.339936,2084.965546,6.503872,4248.289734,0.003552,49.490849,108908.937500
+10,1.296743,1.294896,0.000033,0.402373,6345.264160,82626.662463,3.434560,2020.342789,6.487808,4257.328003,0.003712,49.710911,108908.937500
+11,1.312237,1.294896,0.000036,0.554058,6353.096191,82524.801168,3.506912,2023.778214,6.483936,4260.826904,0.003232,50.503616,108908.937500
+12,1.311145,1.294896,0.000039,0.428505,6334.817871,82762.916104,3.446368,2019.977585,6.588512,4247.066528,0.003136,49.664478,108908.937500
+13,1.326215,1.294896,0.000042,0.456838,6309.560547,83094.218069,3.525888,2018.268127,6.483840,4223.184143,0.003424,50.131489,108908.937500
+14,1.368519,1.294896,0.000045,0.818343,6299.720215,83224.013467,3.477152,2013.747162,6.458400,4218.012939,0.003168,50.018719,108908.937500
+15,1.356700,1.294896,0.000048,0.479156,6285.380371,83413.885723,3.375584,2007.212418,6.482880,4210.481812,0.003712,49.821758,108908.937500
+16,1.342912,1.294896,0.000051,0.526304,6281.114746,83470.533686,3.520384,2006.404953,6.484320,4206.794434,0.003136,49.922272,108908.937500
+17,1.281001,1.294896,0.000054,0.478579,6287.559570,83384.975385,3.368320,2007.988251,6.473184,4212.118530,0.003488,49.638046,108908.937500
+18,1.320383,1.294896,0.000057,0.421508,6258.541016,83771.600872,3.428448,2004.895645,6.469088,4185.912048,0.003744,49.918369,108908.937500
+19,1.339903,1.294896,0.000060,0.526196,6269.643555,83623.254724,3.512096,2005.966019,6.501344,4196.056641,0.003872,49.574978,108908.937500
+20,1.317470,1.294896,0.000063,0.516342,6280.502930,83478.664984,3.608704,2007.289383,6.469920,4204.687195,0.003552,50.428257,108908.937500
+21,1.395066,1.294896,0.000066,0.496802,6266.544434,83664.610625,3.528832,2007.111649,6.480160,4191.755005,0.003552,49.677536,108908.937500
+22,1.330080,1.294896,0.000069,0.410924,6268.903320,83633.128988,3.463200,2002.903122,6.461760,4198.518982,0.003680,49.513344,108908.937500
+23,1.333572,1.294896,0.000072,0.503808,6262.541504,83718.087884,3.435520,2012.880646,6.472672,4182.085815,0.003648,49.533054,108908.937500
+24,1.304763,1.294896,0.000075,0.500245,6271.369629,83600.239026,3.435488,2017.079971,6.473984,4187.040222,0.003616,49.337025,108908.937500
+25,1.386602,1.294896,0.000078,0.460418,6270.594238,83610.576618,3.435424,2011.028763,6.477248,4192.060486,0.003488,49.538784,108908.937500
+26,1.291438,1.294896,0.000081,0.521467,6254.862793,83820.863439,3.370880,2008.536530,6.479040,4179.229004,0.003168,49.273087,108908.937500
+27,1.305346,1.294896,0.000084,0.534433,6256.118652,83804.037157,3.519360,2008.585587,6.470784,4179.299805,0.003616,50.176895,108908.937500
+28,1.341260,1.294896,0.000087,0.436171,6269.882812,83620.063672,3.580960,2002.757919,6.485056,4199.466309,0.003616,49.577278,108908.937500
+29,1.325810,1.294896,0.000090,0.467511,6267.438965,83652.669446,3.423488,2010.482224,6.483104,4189.631958,0.003264,49.424480,108908.937500
+30,1.296540,1.294896,0.000093,0.490903,6261.861816,83727.174980,3.530112,2001.807053,6.490016,4192.541260,0.003264,49.488544,108908.937500
+31,1.373392,1.294896,0.000096,0.503799,6286.106445,83404.251035,3.364704,2006.934891,6.398240,4212.247742,0.003232,49.254177,108908.937500
+32,1.278024,1.294896,0.000099,0.527378,6255.450195,83812.992451,3.375648,2001.516403,6.376928,4186.942566,0.003456,49.325855,108908.937500
+33,1.349225,1.294896,0.000102,0.512099,6301.241699,83203.918375,3.371584,2015.089828,6.391840,4218.194336,0.002912,50.296383,108908.937500
+34,1.328486,1.294896,0.000105,0.518545,6263.756836,83701.844393,3.365888,2003.447174,6.375808,4192.899109,0.002784,49.753056,108908.937500
+35,1.337970,1.294896,0.000108,0.426131,6271.286133,83601.352083,3.389408,2005.766647,6.383200,4198.000061,0.003392,49.826977,108908.937500
+36,1.324051,1.294896,0.000111,0.522886,6279.227051,83495.627051,3.305152,2012.104828,6.407648,4199.832825,0.002848,49.647648,108908.937500
+37,1.324726,1.294896,0.000114,0.563282,6299.068848,83232.619405,3.331136,2013.383911,6.365920,4217.775879,0.003264,50.320255,108908.937500
+38,1.316116,1.294896,0.000117,0.515835,6260.176270,83749.718447,3.340352,2005.016983,6.367808,4187.928833,0.003456,49.594498,108908.937500
+39,1.348061,1.294896,0.000120,0.538212,6284.027344,83431.845745,3.380960,2012.282364,6.374688,4204.635559,0.003680,49.445408,108908.937500
+40,1.304712,1.294896,0.000123,0.522311,6297.126953,83258.286502,3.310816,2007.445953,6.372352,4222.262695,0.003552,49.792290,108908.937500
+41,1.286568,1.294896,0.000126,0.513661,6278.641113,83503.419058,3.204256,2008.486298,6.372768,4202.720276,0.003232,49.937889,108908.937500
+42,1.342654,1.294896,0.000129,0.505271,6289.848145,83354.635589,3.311968,2008.303726,6.382080,4214.437622,0.003104,49.502399,108908.937500
+43,1.350590,1.294896,0.000132,0.502413,6273.005371,83578.439517,3.280768,2006.402496,6.397280,4198.787292,0.003840,50.257824,108908.937500
+44,1.265114,1.294896,0.000135,0.555666,6265.236328,83682.078782,3.378400,2007.791229,6.377504,4189.962708,0.002816,49.824768,108908.937500
+45,1.346877,1.294896,0.000138,0.611855,6295.645508,83277.878233,3.384384,2007.467285,6.367776,4220.921265,0.003232,49.611038,108908.937500
+46,1.335825,1.294896,0.000141,0.560039,6310.762695,83078.389303,3.375712,2013.995758,6.382784,4228.934875,0.003488,50.002750,108908.937500
+47,1.367033,1.294896,0.000144,0.565212,6282.054688,83458.044554,3.282592,2008.075241,6.379008,4206.697632,0.003424,49.754143,108908.937500
+48,1.362973,1.294896,0.000147,0.681856,6286.662109,83396.879119,3.279424,2008.367645,6.389696,4210.907471,0.003360,49.856449,108908.937500
+49,1.340159,1.294896,0.000150,0.545723,6296.073242,83272.220610,3.263360,2010.147202,6.387424,4218.140747,0.003136,50.283615,108908.937500
+50,1.380490,1.294896,0.000153,0.582814,6279.756348,83488.589521,3.308576,2006.882034,6.368704,4205.785583,0.003200,49.541824,108908.937500
+51,1.399287,1.294896,0.000156,0.603464,6277.289551,83521.398170,3.302880,2012.631592,6.376224,4197.315674,0.003520,49.789886,108908.937500
+52,1.412886,1.294896,0.000159,0.564898,6288.651367,83370.498599,3.284704,2004.439270,6.374624,4217.161011,0.002880,49.509216,108908.937500
+53,1.322852,1.294896,0.000162,0.645156,6281.726074,83462.410459,3.375616,2009.010330,6.394560,4205.211182,0.003296,49.917122,108908.937500
+54,1.332037,1.294896,0.000165,0.721773,6320.492188,82950.502025,3.282880,2011.223450,6.371104,4241.983826,0.003040,49.700481,108908.937500
+55,1.426419,1.294896,0.000168,0.530469,6291.801270,83328.760325,3.439840,2012.618057,6.369504,4211.877808,0.003616,49.565151,108908.937500
+56,1.385721,1.294896,0.000171,0.557383,6275.590820,83544.006455,3.328032,2009.857254,6.389184,4198.654114,0.002848,49.454144,108908.937500
+57,1.316328,1.294896,0.000174,0.531537,6300.964355,83207.580685,3.371136,2016.222183,6.371616,4217.629028,0.003136,49.495968,108908.937500
+58,1.336606,1.294896,0.000177,0.557930,6267.410156,83653.053962,3.196096,2007.577530,6.370784,4192.780457,0.003488,49.620289,108908.937500
+59,1.338307,1.294896,0.000180,0.556392,6268.463379,83638.998636,3.403744,2005.646027,6.381824,4194.759705,0.003392,50.374367,108908.937500
+60,1.386156,1.294896,0.000183,0.560389,6274.822266,83554.239117,3.254880,2008.923889,6.366944,4199.144897,0.003520,49.336254,108908.937500
+61,1.312847,1.294896,0.000186,0.783419,6273.959961,83565.722967,3.158464,2006.325760,6.374752,4200.934570,0.003424,49.348225,108908.937500
+62,1.348668,1.294896,0.000189,0.702760,6283.367676,83440.604952,3.219360,2008.241669,6.376608,4208.231567,0.003232,49.505409,108908.937500
+63,1.348439,1.294896,0.000192,0.648666,6282.385254,83453.653160,3.222336,2008.754303,6.382080,4205.943604,0.003264,50.185696,108908.937500
+64,1.311772,1.294896,0.000195,0.522569,6278.653809,83503.250216,3.207200,2010.886993,6.371968,4200.611572,0.003008,49.710655,108908.937500
+65,1.364191,1.294896,0.000198,0.706308,6271.512695,83598.331929,3.160096,2007.072510,6.377664,4197.783142,0.003680,49.293858,108908.937500
+66,1.342329,1.294896,0.000201,0.542978,6286.751465,83395.693775,3.226208,2009.964233,6.369856,4209.386780,0.003040,49.917824,108908.937500
+67,1.387063,1.294896,0.000204,0.562069,6297.752930,83250.010893,3.341984,2013.844910,6.380224,4217.037476,0.003744,49.263744,108908.937500
+68,1.359468,1.294896,0.000207,0.624760,6290.335938,83348.171737,3.336256,2010.720673,6.367904,4211.772522,0.003360,50.251904,108908.937500
+69,1.407157,1.294896,0.000210,0.566290,6284.269043,83428.636873,3.317664,2002.518799,6.372288,4214.272705,0.003648,49.871426,108908.937500
+70,1.360724,1.294896,0.000213,0.647732,6262.198242,83722.676882,3.302560,2003.456619,6.393568,4191.912720,0.003040,49.271168,108908.937500
+71,1.374638,1.294896,0.000216,0.548389,6298.639160,83238.297459,3.202272,2010.826202,6.372064,4220.918152,0.003392,49.463360,108908.937500
+72,1.298399,1.294896,0.000219,0.546931,6283.730469,83435.787484,3.239008,2010.648483,6.372480,4205.964172,0.003168,49.770046,108908.937500
+73,1.381727,1.294896,0.000222,0.693588,6269.743652,83621.919662,3.175968,2004.387405,6.368608,4198.634277,0.003488,49.389378,108908.937500
+74,1.360476,1.294896,0.000225,0.629983,6288.641602,83370.628065,3.283456,2004.689621,6.375456,4216.963196,0.003328,49.464127,108908.937500
+75,1.374706,1.294896,0.000228,0.657585,6284.982422,83419.167280,3.256736,2009.500351,6.379328,4208.495361,0.003328,49.479168,108908.937500
+76,1.324301,1.294896,0.000231,0.968114,6271.949707,83592.507034,3.342496,2004.304596,6.369120,4200.694519,0.003424,49.360958,108908.937500
+77,1.386806,1.294896,0.000234,0.639303,6306.626465,83132.876653,3.251392,2007.471222,6.384608,4232.182495,0.003104,49.421570,108908.937500
+78,1.358803,1.294896,0.000237,0.766064,6298.032227,83246.319031,3.228736,2004.594543,6.392320,4226.416870,0.003104,49.517090,108908.937500
+79,1.369025,1.294896,0.000240,0.724389,6282.020020,83458.505126,3.350400,2013.531830,6.373280,4201.223633,0.003040,49.646942,108908.937500
+80,1.366161,1.294896,0.000243,0.789659,6300.059082,83219.537019,3.405600,2009.185272,6.384448,4223.442139,0.002880,49.756191,108908.937500
+81,1.420741,1.294896,0.000246,0.675051,6276.880371,83526.842795,3.434848,2009.552673,6.381824,4200.250000,0.003072,49.405857,108908.937500
+82,1.449490,1.294896,0.000249,0.650437,6302.695801,83184.722311,3.296832,2013.415665,6.374272,4222.117371,0.003168,49.608097,108908.937500
+83,1.364364,1.294896,0.000252,0.718089,6287.534668,83385.305638,3.173856,2011.442413,6.376064,4209.321594,0.003712,49.298687,108908.937500
+84,1.382751,1.294896,0.000255,0.622832,6275.368164,83546.970679,3.305088,2010.360977,6.368480,4197.766663,0.002976,49.714306,108908.937500
+85,1.382727,1.294896,0.000258,0.689547,6278.229980,83508.887319,3.105760,2008.357697,6.365728,4203.127136,0.003232,49.480289,108908.937500
+86,1.377730,1.294896,0.000261,0.659395,6281.099609,83470.734840,3.227776,2014.882401,6.372800,4199.191528,0.002880,49.669727,108908.937500
+87,1.459780,1.294896,0.000264,0.900990,6281.156738,83469.975650,3.117696,2012.201263,6.389216,4202.108643,0.003616,49.497055,108908.937500
+88,1.464082,1.294896,0.000267,0.879077,6301.083984,83206.000952,3.272288,2013.461655,6.383232,4220.546570,0.003264,49.559807,108908.937500
+89,1.402862,1.294896,0.000270,0.998102,6294.037109,83299.159330,3.393696,2009.135162,6.364384,4217.905396,0.003648,49.375774,108908.937500
+90,1.480830,1.294896,0.000273,0.686547,6269.211914,83629.012257,3.244576,2005.460831,6.370016,4196.530884,0.003360,49.719646,108908.937500
+91,1.418258,1.294896,0.000276,0.624663,6297.325684,83255.659044,3.220448,2010.011353,6.414656,4220.187866,0.003552,49.703808,108908.937500
+92,1.428064,1.294896,0.000279,0.722415,6277.128418,83523.542150,3.314400,2006.914993,6.363968,4203.175659,0.002944,49.467682,108908.937500
+93,1.457621,1.294896,0.000282,0.712933,6277.639160,83516.746762,3.132832,2006.498077,6.378272,4204.099854,0.003424,49.662495,108908.937500
+94,1.319050,1.294896,0.000285,0.565523,6289.237305,83362.731377,3.236320,2010.231155,6.381056,4212.216003,0.003232,49.393887,108908.937500
+95,1.407644,1.294896,0.000288,0.672691,6316.613281,83001.440274,3.142048,2012.163803,6.373632,4237.307373,0.003904,49.763870,108908.937500
+96,1.406497,1.294896,0.000291,0.707097,6289.435059,83360.110267,3.360032,2017.000320,6.365952,4205.271240,0.003360,49.575073,108908.937500
+97,1.403561,1.294896,0.000294,0.663181,6267.925293,83646.178838,3.320736,2003.648834,6.372928,4197.146240,0.003488,49.573025,108908.937500
+98,1.393360,1.294896,0.000297,0.735300,6296.074707,83272.201236,3.372096,2014.023682,6.377600,4214.740356,0.002976,49.665630,108908.937500
+99,1.421664,1.294896,0.000300,0.678905,6272.133301,83590.060169,3.257760,2012.199463,6.363008,4192.648682,0.003072,49.841599,108908.937500
+100,1.442932,1.294896,0.000300,0.671236,6347.870117,82592.742183,3.321888,2136.723206,6.364416,4143.349243,0.003392,50.101727,108908.937500
+101,1.480651,1.294896,0.000300,0.753554,6287.574707,83384.774644,3.349792,2058.598969,6.371264,4161.992920,0.003264,49.343742,108908.937500
+102,1.375014,1.294896,0.000300,0.687758,6285.326172,83414.605012,3.416128,2047.170013,6.358432,4170.541565,0.003008,49.898750,108908.937500
+103,1.441529,1.294896,0.000300,0.746076,6254.865723,83820.824179,3.263584,2004.573593,6.371104,4182.739746,0.003776,50.040642,108908.937500
+104,1.510992,1.294896,0.000300,0.763571,6265.731934,83675.459716,3.327840,2008.128830,6.365664,4190.381958,0.003296,49.548286,108908.937500
+105,1.419842,1.294896,0.000300,0.705664,6281.939941,83459.569001,3.248352,2011.728043,6.383200,4202.956787,0.003360,49.698593,108908.937500
+106,1.435070,1.294896,0.000300,0.874984,6303.385742,83175.617270,3.327552,2018.693619,6.392992,4217.132812,0.003040,49.927265,108908.937500
+107,1.502292,1.294896,0.000300,0.891801,6299.848145,83222.323455,3.291904,2010.119232,6.370080,4222.547363,0.003040,49.673119,108908.937500
+108,1.378002,1.294896,0.000300,0.749525,6299.172363,83231.251625,3.254752,2009.964310,6.372320,4222.335327,0.003008,49.303169,108908.937500
+109,1.419788,1.294896,0.000300,0.673315,6320.151855,82954.968803,3.204704,2057.856766,6.387808,4195.198792,0.003072,49.577377,108908.937500
+110,1.450531,1.294896,0.000299,0.648350,6326.891602,82866.600697,3.261152,2031.118896,6.378112,4228.780273,0.003040,49.389153,108908.937500
+111,1.426576,1.294896,0.000299,0.739924,6314.300293,83031.844492,3.309952,2014.500732,6.398496,4232.701965,0.003680,49.513535,108908.937500
+112,1.403461,1.294896,0.000299,0.680895,6303.802246,83170.121703,3.272128,2020.075470,6.368608,4216.640442,0.003136,49.546783,108908.937500
+113,1.387195,1.294896,0.000299,0.638524,6324.342773,82899.997483,3.306048,2019.036865,6.394656,4238.075134,0.003584,49.588959,108908.937500
+114,1.429039,1.294896,0.000299,0.617910,6282.528809,83451.746259,3.356416,2014.878098,6.371712,4200.405151,0.003104,49.595905,108908.937500
+115,1.412321,1.294896,0.000299,0.827142,6317.997070,82983.261019,3.424544,2017.435654,6.380992,4233.173340,0.003744,49.624928,108908.937500
+116,1.351416,1.294896,0.000299,0.691355,6355.820312,82489.430824,3.271520,2085.558319,6.385120,4202.881348,0.003552,49.624096,108908.937500
+117,1.361993,1.294896,0.000298,0.621928,6347.737793,82594.463902,3.327456,2048.742218,6.407264,4231.934753,0.003552,49.455681,108908.937500
+118,1.438746,1.294896,0.000298,0.607086,6325.866211,82880.032950,3.278048,2038.370926,6.388064,4219.834717,0.003040,50.067650,108908.937500
+119,1.453658,1.294896,0.000298,0.636319,6312.785156,83051.773033,3.249888,2016.719971,6.402400,4229.167603,0.003392,49.331966,108908.937500
+120,1.387286,1.294896,0.000298,0.718123,6318.729004,82973.648605,3.291296,2020.460983,6.397376,4231.107361,0.003456,49.554943,108908.937500
+121,1.415852,1.294896,0.000298,0.619021,6324.865723,82893.143189,3.254464,2022.949677,6.395872,4233.840332,0.003616,50.484863,108908.937500
+122,1.479933,1.294896,0.000297,0.585861,6328.258301,82848.704190,3.245056,2017.541214,6.389376,4243.390259,0.003488,49.768288,108908.937500
+123,1.440239,1.294896,0.000297,0.763344,6311.217285,83072.405261,3.296352,2011.914307,6.389920,4232.291382,0.003328,49.454784,108908.937500
+124,1.466681,1.294896,0.000297,0.677145,6301.902832,83195.189449,3.221184,2011.505493,6.386560,4223.545227,0.002720,49.357311,108908.937500
+125,1.422263,1.294896,0.000297,0.666030,6324.039551,82903.972341,3.233248,2026.550034,6.410336,4230.341614,0.003168,49.611809,108908.937500
+126,1.435130,1.294896,0.000296,0.603647,6311.325195,83070.984900,3.068288,2017.174026,6.368544,4227.489746,0.003008,49.316994,108908.937500
+127,1.478923,1.294896,0.000296,0.600341,6354.746582,82503.368660,3.074976,2072.239532,6.357984,4215.456970,0.003456,49.745377,108908.937500
+128,1.507035,1.294896,0.000296,0.568917,6348.337891,82586.656387,3.209760,2056.715393,6.370144,4224.660339,0.002496,49.437695,108908.937500
+129,1.452367,1.294896,0.000295,0.853240,6329.677734,82830.125324,3.209760,2044.371582,6.375488,4217.989563,0.003424,49.868320,108908.937500
+130,1.569316,1.294896,0.000295,0.726095,6296.595703,83265.311085,3.059232,2016.617752,6.372640,4213.032410,0.003072,49.608768,108908.937500
+131,1.450306,1.294896,0.000295,0.757966,6319.998047,82956.987662,3.216032,2016.893890,6.467808,4235.194580,0.003648,50.199650,108908.937500
+132,1.454688,1.294896,0.000294,0.605309,6312.206055,83059.392462,6.187168,2021.601364,6.364576,4220.527649,0.003392,49.463711,108908.937500
+133,1.431383,1.294896,0.000294,0.593414,6329.631836,82830.725955,3.201440,2023.844177,6.369664,4238.474670,0.002944,49.787041,108908.937500
+134,1.439821,1.294896,0.000294,0.590355,6300.036133,83219.840164,3.220192,2011.740311,6.365984,4221.252197,0.002784,49.564705,108908.937500
+135,1.477309,1.294896,0.000293,0.597530,6370.561523,82298.553757,3.150016,2075.320038,6.381344,4228.247192,0.003168,49.571522,108908.937500
+136,1.419291,1.294896,0.000293,0.659685,6316.178711,83007.151000,3.307360,2016.132126,6.371904,4233.018372,0.003104,49.402912,108908.937500
+137,1.411285,1.294896,0.000293,0.648018,6292.219238,83323.225105,3.237632,2013.106110,6.358592,4212.247009,0.003136,49.261986,108908.937500
+138,1.478121,1.294896,0.000292,0.530393,6332.395020,82794.582205,3.260256,2025.982834,6.365728,4239.592285,0.003264,49.197792,108908.937500
+139,1.480008,1.294896,0.000292,0.547857,6302.456055,83187.886667,3.266528,2008.849838,6.365696,4226.391235,0.003584,49.656929,108908.937500
+140,1.493623,1.294896,0.000291,0.550624,6304.767090,83157.393846,3.268736,2020.935837,6.397536,4216.818298,0.002848,49.353119,108908.937500
+141,1.482856,1.294896,0.000291,0.608232,6341.750000,82672.448457,3.105600,2045.806168,6.386816,4229.184753,0.003488,49.349823,108908.937500
+142,1.466891,1.294896,0.000290,0.579300,6307.547852,83120.732865,3.237024,2018.308990,6.375456,4222.378113,0.002848,49.276417,108908.937500
+143,1.405515,1.294896,0.000290,0.593449,6327.348145,82860.621547,3.266752,2022.629471,6.370016,4237.425720,0.003616,49.708672,108908.937500
+144,1.492733,1.294896,0.000290,0.617969,6321.729004,82934.273152,3.283680,2019.385010,6.369088,4235.304260,0.003104,49.440449,108908.937500
+145,1.393695,1.294896,0.000289,0.612817,6305.765137,83144.232085,3.313600,2014.125381,6.366560,4224.486328,0.003200,49.389633,108908.937500
+146,1.406142,1.294896,0.000289,0.475764,6302.262695,83190.438950,3.373504,2014.197586,6.351232,4220.809631,0.003168,49.540958,108908.937500
+147,1.404284,1.294896,0.000288,0.494678,6333.325684,82782.415779,3.346368,2017.468475,6.471904,4248.564392,0.003104,49.498463,108908.937500
+148,1.456305,1.294896,0.000288,0.521166,6307.479492,83121.633713,3.240608,2013.445847,6.362816,4226.928284,0.002880,49.493248,108908.937500
+149,1.452610,1.294896,0.000287,0.496439,6310.163086,83086.283644,3.372672,2011.704483,6.363264,4231.243225,0.002848,49.508896,108908.937500
+150,1.402426,1.294896,0.000287,0.550255,6293.218262,83309.997873,3.409312,2014.746078,6.370528,4211.187866,0.003008,49.521824,108908.937500
+151,1.426098,1.294896,0.000286,0.517503,6304.621582,83159.313081,3.337920,2012.552582,6.373856,4224.862976,0.003456,49.490623,108908.937500
+152,1.476027,1.294896,0.000286,0.600313,6305.911621,83142.300670,3.283200,2013.442810,6.364224,4225.493347,0.003168,49.380447,108908.937500
+153,1.428901,1.294896,0.000285,0.504415,6333.245605,82783.462487,3.240768,2022.725021,6.372992,4242.927063,0.003488,49.734241,108908.937500
+154,1.461169,1.294896,0.000284,0.572215,6284.951172,83419.582056,3.178048,2011.650116,6.374432,4206.238892,0.003296,49.614178,108908.937500
+155,1.442151,1.294896,0.000284,0.561559,6307.083984,83126.846146,3.288800,2016.711060,6.363520,4223.201416,0.003552,49.514305,108908.937500
+156,1.428072,1.294896,0.000283,0.485563,6349.765137,82568.093262,3.203360,2049.651779,6.379424,4233.147827,0.002624,49.360832,108908.937500
+157,1.488164,1.294896,0.000283,0.493952,6319.095215,82968.840028,3.154112,2033.790253,6.376864,4218.575378,0.002848,49.291328,108908.937500
+158,1.497399,1.294896,0.000282,0.487407,6351.240234,82548.916535,3.183072,2063.437912,6.371520,4220.370911,0.002944,49.901505,108908.937500
+159,1.444991,1.294896,0.000282,0.487232,6305.076172,83153.317376,3.159936,2012.416916,6.380992,4225.415771,0.003744,49.765282,108908.937500
+160,1.488986,1.294896,0.000281,0.520883,6317.526367,82989.443894,3.258432,2030.461227,6.385664,4219.303101,0.002944,50.172382,108908.937500
+161,1.451132,1.294896,0.000280,0.476776,6315.287109,83018.870072,3.221024,2013.605026,6.365088,4234.430847,0.003232,49.684353,108908.937500
+162,1.445982,1.294896,0.000280,0.546664,6320.672363,82948.137455,3.198208,2010.061844,6.614752,4242.899414,0.002912,49.966400,108908.937500
+163,1.454782,1.294896,0.000279,0.453941,6302.448730,83187.983341,3.175232,2015.935822,6.370752,4219.382629,0.002976,49.648415,108908.937500
+164,1.377520,1.294896,0.000278,0.642361,6300.221191,83217.395719,3.187104,2012.559357,6.375072,4220.595642,0.003008,49.567551,108908.937500
+165,1.443390,1.294896,0.000278,0.504592,6339.562012,82700.981398,3.293216,2038.047989,6.375136,4234.168518,0.003072,49.586433,108908.937500
+166,1.443651,1.294896,0.000277,0.454753,6303.098145,83179.412406,3.222272,2020.985413,6.391616,4215.248108,0.003072,49.267040,108908.937500
+167,1.459447,1.294896,0.000276,0.493057,6321.603027,82935.925861,3.369824,2039.939911,6.379808,4214.083374,0.003168,49.885761,108908.937500
+168,1.499442,1.294896,0.000276,0.474921,6321.226562,82940.865165,3.229920,2017.586060,6.389312,4236.498352,0.003200,49.571743,108908.937500
+169,1.408868,1.294896,0.000275,0.535967,6332.744629,82790.011397,3.104288,2042.840012,6.358848,4222.718994,0.003520,49.808098,108908.937500
+170,1.415292,1.294896,0.000274,0.462294,6357.738281,82464.545851,3.220800,2075.409164,6.413472,4214.908447,0.003616,49.720512,108908.937500
+171,1.486619,1.294896,0.000274,0.420980,6342.415527,82663.773406,3.192096,2061.940460,6.379392,4213.722290,0.003552,49.276417,108908.937500
+172,1.444242,1.294896,0.000273,0.469536,6311.655762,83066.634144,3.302080,2045.133102,6.353888,4199.404724,0.003008,49.443489,108908.937500
+173,1.427189,1.294896,0.000272,0.426014,6352.709473,82529.824834,3.268672,2071.770844,6.405056,4213.259644,0.002944,49.757854,108908.937500
+174,1.450822,1.294896,0.000271,0.408133,6326.206543,82875.574239,3.218944,2027.276886,6.381120,4232.128418,0.002816,49.227425,108908.937500
+175,1.438023,1.294896,0.000271,0.565074,6350.000000,82565.039370,3.295296,2054.077469,6.381376,4228.617615,0.003136,49.649632,108908.937500
+176,1.403051,1.294896,0.000270,0.574628,6317.023926,82996.044682,3.176704,2012.843842,6.369536,4237.246460,0.002720,49.531456,108908.937500
+177,1.494423,1.294896,0.000269,0.448123,6300.301270,83216.338008,3.288448,2014.582169,6.376544,4218.477966,0.003424,49.644066,108908.937500
+178,1.461557,1.294896,0.000268,0.445583,6308.250977,83111.468131,3.206368,2009.682465,6.369056,4231.218506,0.002976,49.836704,108908.937500
+179,1.462278,1.294896,0.000268,0.570873,6316.369629,83004.642034,3.255776,2016.237534,6.397440,4233.114990,0.003200,49.417854,108908.937500
+180,1.506178,1.294896,0.000267,0.405512,6298.740723,83236.955303,3.166016,2012.014618,6.369472,4220.030945,0.003520,49.283710,108908.937500
+181,1.408684,1.294896,0.000266,0.392813,6313.124023,83047.315094,3.253600,2016.401443,6.376960,4229.736023,0.002976,49.463711,108908.937500
+182,1.409247,1.294896,0.000265,0.511128,6349.642090,82569.693312,3.224064,2067.056808,6.359744,4215.661926,0.003232,49.379520,108908.937500
+183,1.442007,1.294896,0.000265,0.413309,6357.687988,82465.198193,3.349824,2078.160843,6.373344,4212.343445,0.003392,49.394302,108908.937500
+184,1.444275,1.294896,0.000264,0.499370,6335.668945,82751.798512,3.475008,2030.860199,6.371456,4237.490112,0.002848,49.498753,108908.937500
+185,1.408447,1.294896,0.000263,0.470220,6362.892578,82397.744982,3.170560,2067.494492,6.374912,4227.798767,0.003008,50.094433,108908.937500
+186,1.479656,1.294896,0.000262,0.423814,6302.049316,83193.255666,3.343616,2018.352951,6.379680,4215.685120,0.003040,50.236961,108908.937500
+187,1.460944,1.294896,0.000261,0.431184,6305.985352,83141.328559,3.311520,2014.561264,6.376096,4223.988159,0.003424,49.717857,108908.937500
+188,1.431488,1.294896,0.000260,0.453643,6294.174805,83297.337025,3.198912,2018.662338,6.362144,4208.374512,0.003552,49.651810,108908.937500
+189,1.444559,1.294896,0.000260,0.412251,6299.068359,83232.625856,3.163072,2015.504745,6.383840,4216.366272,0.003392,49.748993,108908.937500
+190,1.415559,1.294896,0.000259,0.400295,6316.997559,82996.391108,3.258720,2018.852402,6.375520,4230.600220,0.003104,49.856224,108908.937500
+191,1.441320,1.294896,0.000258,0.386981,6297.479004,83253.632076,3.239584,2018.344574,6.364704,4212.257629,0.003392,49.373730,108908.937500
+192,1.424655,1.294896,0.000257,0.493587,6303.411621,83175.275790,3.124064,2012.840942,6.369376,4223.929749,0.002976,49.264290,108908.937500
+193,1.420979,1.294896,0.000256,0.376018,6322.462891,82924.646466,3.077792,2021.630035,6.373120,4234.083435,0.003104,49.381023,108908.937500
+194,1.427058,1.294896,0.000255,0.417725,6296.949219,83260.636506,3.284416,2010.007172,6.409920,4219.775208,0.003488,49.490337,108908.937500
+195,1.410106,1.294896,0.000254,0.366513,6322.181641,82928.335471,3.141856,2020.234985,6.390304,4235.054321,0.002976,49.439552,108908.937500
+196,1.435013,1.294896,0.000254,0.628455,6344.437500,82637.428456,3.299200,2077.771851,6.376512,4199.037109,0.002752,49.912830,108908.937500
+197,1.451669,1.294896,0.000253,0.452651,6354.952148,82500.699888,3.300896,2063.846771,6.391328,4223.996399,0.002848,49.436447,108908.937500
+198,1.398883,1.294896,0.000252,0.407946,6320.272949,82953.379421,3.249376,2013.690369,6.478432,4238.997986,0.002848,49.837505,108908.937500
+199,1.404828,1.294896,0.000251,0.384534,6330.377930,82820.963586,3.347488,2018.928558,6.409568,4244.103455,0.003296,49.404095,108908.937500
+200,1.403817,1.294896,0.000250,0.382767,6724.157715,77970.806491,3.021664,2514.863007,6.377408,4141.847504,0.002720,49.796638,108908.937500
+201,1.438127,1.294896,0.000249,0.319228,6242.054199,83992.862488,3.467168,2001.669739,6.357376,4173.120361,0.003296,49.514271,108908.937500
+202,1.406299,1.294896,0.000248,0.394685,6248.999023,83899.517032,3.189184,2001.923279,6.349536,4179.734680,0.003200,49.887329,108908.937500
+203,1.497176,1.294896,0.000247,0.432060,6287.291992,83388.524130,3.170848,2010.599319,6.362880,4209.979614,0.003168,49.260098,108908.937500
+204,1.398200,1.294896,0.000246,0.336504,6285.028320,83418.558084,3.520288,2014.433777,6.478880,4203.152771,0.003712,49.387810,108908.937500
+205,1.364092,1.294896,0.000245,0.368072,6301.304688,83203.086662,3.316576,2018.974869,6.504384,4214.846252,0.003520,49.490849,108908.937500
+206,1.459170,1.294896,0.000244,0.392012,6362.353516,82404.726288,3.543328,2075.090408,6.523264,4219.642944,0.003616,49.491135,108908.937500
+207,1.484398,1.294896,0.000244,0.350218,6342.229980,82666.191799,3.414848,2059.903824,6.533792,4214.473816,0.003744,49.812416,108908.937500
+208,1.348089,1.294896,0.000243,0.391527,6355.720703,82490.723631,3.490432,2069.152420,6.506304,4218.920105,0.003520,49.527679,108908.937500
+209,1.449291,1.294896,0.000242,0.391850,6319.395996,82964.891000,3.421760,2021.657532,6.505824,4229.925598,0.003776,49.813503,108908.937500
+210,1.460217,1.294896,0.000241,0.504283,6311.211914,83072.475959,3.335136,2015.451782,6.515552,4228.495728,0.003360,49.362144,108908.937500
+211,1.480269,1.294896,0.000240,0.432505,6329.082520,82837.915035,3.416064,2019.682419,6.505632,4241.764465,0.003808,49.588161,108908.937500
+212,1.416566,1.294896,0.000239,0.377926,6321.432617,82938.161608,3.493728,2022.364990,6.543648,4231.263794,0.003456,49.763744,108908.937500
+213,1.433720,1.294896,0.000238,0.658322,6307.520508,83121.093201,3.336288,2013.653748,6.493088,4226.467834,0.003648,49.513153,108908.937500
+214,1.418691,1.294896,0.000237,0.331679,6316.352051,83004.873032,3.610560,2020.916748,6.540288,4227.680664,0.003424,49.457890,108908.937500
+215,1.428402,1.294896,0.000236,0.361554,6316.665527,83000.753757,3.569024,2023.857880,6.533760,4224.757874,0.003680,49.761726,108908.937500
+216,1.396070,1.294896,0.000235,0.346104,6352.212402,82536.282919,3.349120,2065.028793,6.509152,4219.909424,0.003584,49.292961,108908.937500
+217,1.417023,1.294896,0.000234,0.391398,6346.337402,82612.689298,3.366944,2049.421432,6.494432,4229.389038,0.003776,49.586624,108908.937500
+218,1.450046,1.294896,0.000233,0.299446,6354.795410,82502.734732,3.326752,2080.747131,6.526432,4206.547485,0.003744,49.545761,108908.937500
+219,1.475388,1.294896,0.000232,0.348860,6335.395508,82755.370103,3.396896,2026.135971,6.514816,4241.889587,0.003424,49.414623,108908.937500
+220,1.406461,1.294896,0.000231,0.426121,6305.403320,83149.003064,3.449024,2017.707504,6.498208,4220.165588,0.003584,49.564610,108908.937500
+221,1.394355,1.294896,0.000230,0.430230,6305.007812,83154.218931,3.441856,2018.766327,6.529120,4218.517273,0.003552,49.728642,108908.937500
+222,1.381861,1.294896,0.000229,0.306968,6324.836426,82893.527153,3.519904,2018.478912,6.480320,4238.211914,0.003520,50.083614,108908.937500
+223,1.416641,1.294896,0.000228,0.430763,6309.973145,83088.784689,3.520224,2014.582199,6.508288,4227.431885,0.003232,49.729504,108908.937500
+224,1.357221,1.294896,0.000227,0.449955,6306.857910,83129.825892,3.354368,2012.895706,6.492960,4226.083984,0.003680,49.966080,108908.937500
+225,1.454751,1.294896,0.000226,0.356241,6348.482422,82584.776197,3.494176,2066.704544,6.514976,4214.246582,0.003712,49.481792,108908.937500
+226,1.376981,1.294896,0.000225,0.322374,6345.981445,82617.323186,3.597792,2055.002884,6.514240,4223.219116,0.003936,49.477791,108908.937500
+227,1.365093,1.294896,0.000224,0.351853,6324.656250,82895.888611,3.693184,2031.624039,6.506656,4225.393494,0.003552,49.376575,108908.937500
+228,1.433433,1.294896,0.000223,0.320631,6306.059570,83140.350032,3.654560,2023.128906,6.512544,4215.249878,0.003200,49.257343,108908.937500
+229,1.400847,1.294896,0.000222,0.321523,6314.377930,83030.823596,3.566592,2030.239929,6.500096,4215.887695,0.003104,50.022114,108908.937500
+230,1.393643,1.294896,0.000221,0.335114,6307.752441,83118.036871,3.399936,2019.481949,6.489440,4220.854248,0.003968,49.447071,108908.937500
+231,1.379871,1.294896,0.000220,0.344076,6332.348145,82795.195089,3.439552,2033.665955,6.495968,4230.935059,0.003776,49.785248,108908.937500
+232,1.477580,1.294896,0.000219,0.388653,6331.771484,82802.735584,3.430240,2017.514481,6.486752,4246.688843,0.003424,49.589088,108908.937500
+233,1.401012,1.294896,0.000218,0.321659,6325.288086,82887.608102,3.492672,2013.419815,6.708032,4244.010620,0.003648,49.567295,108908.937500
+234,1.425210,1.294896,0.000217,0.461593,6327.961426,82852.591020,3.420256,2015.295685,6.558784,4245.097778,0.003104,49.488831,108908.937500
+235,1.462096,1.294896,0.000216,0.318412,6304.928711,83155.262183,3.387840,2013.775879,6.495424,4223.051270,0.004032,50.131840,108908.937500
+236,1.431871,1.294896,0.000215,0.326655,6312.999023,83048.959465,3.600160,2010.440796,6.497792,4234.898560,0.003392,49.371231,108908.937500
+237,1.403202,1.294896,0.000214,0.319853,6306.595215,83133.288587,3.413408,2015.444336,6.503328,4223.621033,0.003776,49.509953,108908.937500
+238,1.356215,1.294896,0.000213,0.285623,6316.456055,83003.506311,3.383104,2017.366425,6.517184,4231.428406,0.003584,49.522625,108908.937500
+239,1.371449,1.294896,0.000211,0.282268,6302.885254,83182.221932,3.387456,2015.192688,6.516608,4220.069580,0.003584,49.647102,108908.937500
+240,1.400371,1.294896,0.000210,0.378662,6313.619141,83040.802481,3.474080,2016.372238,6.528736,4229.781189,0.003552,49.408287,108908.937500
+241,1.377639,1.294896,0.000209,0.311359,6324.975098,82891.709755,3.400352,2015.276123,6.515392,4242.142761,0.003840,49.590305,108908.937500
+242,1.425816,1.294896,0.000208,0.346512,6319.454590,82964.121752,3.393088,2022.573196,6.498176,4228.943115,0.003648,49.976833,108908.937500
+243,1.382616,1.294896,0.000207,0.308766,6315.890625,83010.937195,3.448832,2030.299683,6.494016,4218.179871,0.003168,49.488354,108908.937500
+244,1.416425,1.294896,0.000206,0.319418,6317.643066,82987.910917,3.435552,2013.884781,6.502784,4236.246277,0.003680,49.453602,108908.937500
+245,1.412601,1.294896,0.000205,0.299741,6314.211914,83033.006674,3.637312,2011.909698,6.529504,4234.480042,0.003232,49.464607,108908.937500
+246,1.393735,1.294896,0.000204,0.275566,6340.521973,82688.460392,3.502880,2029.037521,6.552704,4243.968506,0.003680,49.276512,108908.937500
+247,1.422358,1.294896,0.000203,0.288804,6321.202637,82941.179097,3.540448,2022.063553,6.505152,4231.388611,0.003872,49.634689,108908.937500
+248,1.395848,1.294896,0.000202,0.342834,6310.270020,83084.875667,3.391488,2019.793442,6.515680,4222.879211,0.003200,49.603073,108908.937500
+249,1.390024,1.294896,0.000201,0.303338,6342.110352,82667.751101,3.408160,2042.290405,6.523168,4232.141541,0.003264,49.638687,108908.937500
+250,1.370249,1.401836,0.000200,0.290149,11439.264648,45832.316684,3.324896,2020.989746,6.514656,4232.382568,0.003840,49.437695,108908.937500
+251,1.419143,1.401836,0.000199,0.348320,6309.676758,83092.687649,3.477408,2015.331818,6.502944,4226.388245,0.003872,49.876289,108908.937500
+252,1.425415,1.401836,0.000198,0.240878,6340.982910,82682.449618,3.417792,2065.686768,6.486592,4207.662231,0.003488,49.687199,108908.937500
+253,1.396258,1.401836,0.000197,0.385465,6333.112305,82785.204932,3.531104,2017.198166,6.506784,4247.870911,0.003648,50.003902,108908.937500
+254,1.413303,1.401836,0.000196,0.275100,6317.275391,82992.740949,3.371232,2010.392044,6.717472,4238.755676,0.003552,50.021057,108908.937500
+255,1.425304,1.401836,0.000195,0.249878,6325.382812,82886.366808,3.389760,2018.617310,6.504736,4239.456543,0.003328,49.446369,108908.937500
+256,1.367393,1.401836,0.000194,0.248182,6322.178711,82928.373899,3.419584,2021.301010,6.513984,4233.372253,0.003872,49.510914,108908.937500
+257,1.370168,1.401836,0.000193,0.362206,6324.869629,82893.091994,3.671968,2024.829926,6.518048,4232.118408,0.003168,49.581215,108908.937500
+258,1.373518,1.401836,0.000192,0.247201,6309.000000,83101.600888,3.545856,2022.959366,6.514208,4218.242676,0.003296,49.704063,108908.937500
+259,1.361786,1.401836,0.000191,0.231847,6315.234375,83019.563308,3.452256,2016.097992,6.511040,4231.113037,0.003840,50.018784,108908.937500
+260,1.375107,1.401836,0.000190,0.227227,6370.121094,82304.243873,3.357504,2077.750626,6.505984,4224.919495,0.003744,49.535488,108908.937500
+261,1.364820,1.401836,0.000189,0.359953,6340.715820,82685.932450,3.350336,2063.987274,6.512512,4209.567688,0.003872,49.161823,108908.937500
+262,1.402372,1.401836,0.000187,0.244883,6367.981445,82331.898185,3.386112,2084.171219,6.492640,4216.154663,0.003776,49.699490,108908.937500
+263,1.414258,1.401836,0.000186,0.207556,6363.574707,82388.912543,3.413344,2073.921722,6.513792,4222.113037,0.003968,49.550655,108908.937500
+264,1.417034,1.401836,0.000185,0.252772,6306.013184,83140.961609,3.409824,2014.846725,6.517536,4223.502319,0.003456,49.657345,108908.937500
+265,1.355837,1.401836,0.000184,0.256470,6316.670898,83000.683181,3.397184,2026.004547,6.527360,4222.726501,0.003520,49.940865,108908.937500
+266,1.322376,1.401836,0.000183,0.218630,6337.698730,82725.295458,14.124448,2024.059006,6.516448,4235.216492,0.003648,49.723358,108908.937500
+267,1.440838,1.401836,0.000182,0.316766,6305.520020,83147.464186,3.563712,2014.555557,6.497024,4223.238831,0.003200,49.613121,108908.937500
+268,1.420192,1.401836,0.000181,0.256450,6306.085938,83140.002403,3.362784,2013.952347,6.495392,4224.518250,0.003776,49.715137,108908.937500
+269,1.392399,1.401836,0.000180,0.209512,6304.855469,83156.228180,3.473728,2013.598877,6.509568,4223.731934,0.003776,49.417759,108908.937500
+270,1.346640,1.401836,0.000179,0.273303,6351.621582,82543.960346,3.371360,2061.020493,6.519040,4222.460938,0.003776,50.140865,108908.937500
+271,1.433407,1.401836,0.000178,0.233210,6347.447754,82598.237958,3.432160,2047.366974,6.528800,4232.156555,0.003712,49.905407,108908.937500
+272,1.397035,1.401836,0.000177,0.250105,6298.574707,83239.149234,3.399488,2018.038147,6.519168,4212.931091,0.003552,49.583359,108908.937500
+273,1.466986,1.401836,0.000176,0.223607,6322.122070,82929.116864,3.500128,2022.083832,6.510560,4232.238403,0.003840,49.655106,108908.937500
+274,1.389561,1.401836,0.000175,0.274745,6304.074707,83166.527106,3.417536,2019.963715,6.512704,4216.482849,0.003712,49.596066,108908.937500
+275,1.431594,1.401836,0.000174,0.294196,6325.799805,82880.902998,3.453600,2018.137100,6.485536,4240.009949,0.003840,49.657150,108908.937500
+276,1.361612,1.401836,0.000173,0.231768,6334.204102,82770.935637,3.491680,2021.885712,6.525216,4244.687378,0.003680,49.522690,108908.937500
+277,1.433590,1.401836,0.000172,0.251492,6312.011230,83061.956143,3.320832,2015.815765,6.502848,4228.787048,0.003712,49.502014,108908.937500
+278,1.356968,1.401836,0.000171,0.214128,6316.088867,83008.331742,3.419424,2009.564316,6.510464,4238.840820,0.003168,49.684608,108908.937500
+279,1.454445,1.401836,0.000170,0.226587,6315.137207,83020.840690,3.501696,2017.485947,6.526848,4229.900940,0.003168,49.597378,108908.937500
+280,1.379414,1.401836,0.000169,0.303276,6292.586914,83318.356530,3.395872,2013.492187,6.493728,4211.824890,0.003360,49.292706,108908.937500
+281,1.386713,1.401836,0.000168,0.323739,6309.568359,83094.115182,3.521504,2013.854736,6.518592,4227.917480,0.003648,49.597153,108908.937500
+282,1.383668,1.401836,0.000167,0.280347,6297.617188,83251.805308,3.535456,2012.729843,6.530752,4217.054871,0.003296,49.573792,108908.937500
+283,1.299441,1.401836,0.000166,0.242958,6325.676270,82882.521593,3.459328,2024.211121,6.532544,4233.895508,0.003680,49.508320,108908.937500
+284,1.425351,1.401836,0.000165,0.380601,6380.711914,82167.633810,3.372608,2078.053314,6.528992,4234.628662,0.003840,50.064705,108908.937500
+285,1.390065,1.401836,0.000164,0.214712,6321.004883,82943.773928,3.382176,2026.636826,6.485088,4226.620117,0.003328,49.778881,108908.937500
+286,1.374380,1.401836,0.000163,0.244781,6319.970703,82957.346581,3.439680,2017.836365,6.532288,4233.981140,0.003168,50.100769,108908.937500
+287,1.316681,1.401836,0.000162,0.218809,6310.709961,83079.083533,3.456992,2016.683411,6.515520,4226.632263,0.003776,49.348034,108908.937500
+288,1.351033,1.401836,0.000161,0.238247,6331.871094,82801.432979,3.380480,2014.826233,6.675616,4249.294312,0.003488,49.604927,108908.937500
+289,1.321794,1.401836,0.000160,0.179582,6327.123535,82863.563053,3.583552,2017.105316,6.511776,4242.436584,0.003456,49.375614,108908.937500
+290,1.298964,1.401836,0.000159,0.296632,6300.593262,83212.481464,3.538208,2018.200256,6.483744,4214.971008,0.003776,49.257023,108908.937500
+291,1.376741,1.401836,0.000158,0.257554,6312.185059,83059.668741,3.371008,2012.851608,6.500192,4231.782227,0.003136,49.590240,108908.937500
+292,1.383591,1.401836,0.000157,0.202386,6329.681641,82830.074207,3.474560,2018.122452,6.543616,4243.902405,0.003392,49.468063,108908.937500
+293,1.401507,1.401836,0.000156,0.213642,6295.289062,83282.593507,3.334144,2017.074493,6.500256,4210.850830,0.003872,49.436543,108908.937500
+294,1.331201,1.401836,0.000156,0.224216,6322.258301,82927.329928,3.478976,2018.503418,6.508608,4236.221008,0.003296,49.508480,108908.937500
+295,1.387356,1.401836,0.000155,0.216004,6293.031738,83312.467155,3.438784,2016.061493,6.507136,4209.222107,0.003456,49.726849,108908.937500
+296,1.405216,1.401836,0.000154,0.244223,6307.820801,83117.136101,3.443680,2008.984909,6.486432,4231.520325,0.003584,49.279778,108908.937500
+297,1.327925,1.401836,0.000153,0.226844,6299.629395,83225.213289,3.468544,2009.000305,6.533664,4223.049438,0.003616,49.510273,108908.937500
+298,1.343456,1.401836,0.000152,0.218095,6361.321289,82418.097778,3.391936,2081.809372,6.495488,4211.966919,0.003840,49.608704,108908.937500
+299,1.341276,1.401836,0.000151,0.315062,6364.260254,82380.037755,3.380704,2079.995087,6.510944,4216.297546,0.003648,49.973022,108908.937500
+300,1.391191,1.401836,0.000150,0.216005,6192.439453,84665.825798,3.438272,1989.109650,6.456192,4135.535797,0.003648,49.679424,108908.937500
+301,1.362000,1.401836,0.000149,0.212352,6243.517090,83973.182496,3.445152,1998.715546,6.515808,4177.487122,0.003808,49.280930,108908.937500
+302,1.361197,1.401836,0.000148,0.190069,6241.757812,83996.850847,3.328768,2001.488693,6.495488,4173.031921,0.003584,49.293728,108908.937500
+303,1.277367,1.401836,0.000147,0.398156,6303.335938,83176.274468,3.499616,2032.478958,6.528032,4203.260559,0.003744,49.465889,108908.937500
+304,1.388580,1.401836,0.000146,0.246818,6310.978027,83075.554649,3.458432,2019.796097,6.503520,4223.494324,0.003840,49.677406,108908.937500
+305,1.406972,1.401836,0.000146,0.227498,6287.478516,83386.050338,3.443168,2011.755203,6.519232,4208.330200,0.003744,49.355167,108908.937500
+306,1.317721,1.401836,0.000145,0.205253,6337.802246,82723.944302,3.474752,2043.371780,6.533568,4226.769287,0.003616,49.576672,108908.937500
+307,1.430418,1.401836,0.000144,0.230601,6327.149902,82863.217735,3.465088,2024.869019,6.501376,4234.759583,0.003456,49.442688,108908.937500
+308,1.414106,1.401836,0.000143,0.177613,6371.332031,82288.601101,3.419328,2080.672302,6.484352,4222.935059,0.003424,49.708927,108908.937500
+309,1.356122,1.401836,0.000142,0.198559,6321.463379,82937.758012,3.449792,2023.918686,6.530080,4229.985718,0.004032,49.499233,108908.937500
+310,1.299470,1.401836,0.000141,0.197860,6334.440430,82767.847582,3.311680,2015.882416,6.498240,4251.357971,0.003456,49.361023,108908.937500
+311,1.377680,1.401836,0.000140,0.227397,6345.935059,82617.927092,3.400864,2025.641525,6.508160,4252.827820,0.003392,49.460094,108908.937500
+312,1.333104,1.401836,0.000140,0.231579,6324.442383,82898.691816,3.411744,2020.078461,6.518816,4236.813965,0.003584,49.631264,108908.937500
+313,1.316221,1.401836,0.000139,0.266611,6333.140137,82784.841119,3.469632,2022.678314,6.479680,4242.989563,0.003168,49.434273,108908.937500
+314,1.370312,1.401836,0.000138,0.219937,6329.233887,82835.933919,3.431424,2023.169189,6.488448,4238.222168,0.003168,49.847519,108908.937500
+315,1.381422,1.401836,0.000137,0.206535,6326.266113,82874.793853,3.438720,2019.162506,6.505152,4239.690491,0.003456,49.378559,108908.937500
+316,1.354267,1.401836,0.000136,0.209009,6325.813965,82880.717472,3.453536,2019.963730,6.499872,4238.072205,0.003520,49.774815,108908.937500
+317,1.348798,1.401836,0.000135,0.185713,6362.954590,82396.941955,3.392224,2044.779343,6.512768,4250.765320,0.003712,49.426529,108908.937500
+318,1.314881,1.401836,0.000135,0.193838,6333.475098,82780.462845,3.512960,2020.920395,6.504960,4244.928894,0.003584,49.535873,108908.937500
+319,1.435550,1.401836,0.000134,0.180660,6356.310059,82483.075112,3.486720,2025.902420,6.524384,4262.686096,0.003200,49.445473,108908.937500
+320,1.369530,1.401836,0.000133,0.223251,6344.010254,82642.993787,3.392064,2030.147339,6.521120,4246.486511,0.003680,49.341824,108908.937500
+321,1.434570,1.401836,0.000132,0.204615,6321.865234,82932.485993,3.416736,2019.773132,6.499232,4234.454468,0.003712,49.686176,108908.937500
+322,1.364758,1.401836,0.000132,0.201745,6318.532227,82976.232644,3.449056,2017.242691,6.510496,4233.719604,0.003520,49.516930,108908.937500
+323,1.356215,1.401836,0.000131,0.182202,6366.752441,82347.791095,3.443968,2031.024033,6.538080,4268.153137,0.003488,49.474625,108908.937500
+324,1.334546,1.401836,0.000130,0.216080,6325.444824,82885.554229,3.538976,2018.632446,6.511264,4239.198425,0.003936,49.507072,108908.937500
+325,1.355955,1.401836,0.000129,0.267671,6320.625977,82948.746207,3.490784,2022.643799,6.510368,4230.565491,0.003840,49.330688,108908.937500
+326,1.415459,1.401836,0.000129,0.409565,6338.026367,82721.019072,3.485344,2021.344925,6.545696,4249.241516,0.003168,49.307583,108908.937500
+327,1.334630,1.401836,0.000128,0.238010,6323.203125,82914.938780,3.399328,2018.107666,6.517632,4237.367249,0.003264,49.676769,108908.937500
+328,1.298984,1.401836,0.000127,0.739488,6326.005859,82878.203349,3.463104,2018.354492,6.514336,4240.056580,0.003200,49.555969,108908.937500
+329,1.425144,1.401836,0.000126,0.202469,6331.024414,82812.506430,3.367808,2021.071579,6.514016,4242.683655,0.003392,49.278271,108908.937500
+330,1.361766,1.401836,0.000126,0.250317,6312.859375,83050.796613,3.378048,2015.478424,6.511392,4230.050842,0.003776,49.365311,108908.937500
+331,1.340282,1.401836,0.000125,0.172544,6353.839355,82515.148821,3.399072,2023.378418,6.547360,4262.584412,0.003360,49.863007,108908.937500
+332,1.311442,1.401836,0.000124,0.171598,6309.966309,83088.874704,3.414176,2018.250137,6.508224,4224.010132,0.003616,49.697952,108908.937500
+333,1.344754,1.401836,0.000124,0.187984,6345.217285,82627.272864,3.402368,2041.011490,6.537792,4236.766296,0.003776,49.407135,108908.937500
+334,1.337159,1.401836,0.000123,0.156957,6350.102539,82563.706141,3.540288,2056.309769,6.499264,4226.145935,0.003936,49.529282,108908.937500
+335,1.393037,1.401836,0.000122,0.210795,6334.409668,82768.249526,4.126208,2034.900970,6.521248,4231.091797,0.014976,49.707008,108908.937500
+336,1.353839,1.401836,0.000122,0.178954,6376.777344,82218.332511,3.347136,2088.797699,6.506048,4220.650818,0.003616,49.436481,108908.937500
+337,1.370545,1.401836,0.000121,0.164614,6317.617676,82988.244447,3.393216,2019.577942,6.512064,4230.516541,0.003584,49.582878,108908.937500
+338,1.392171,1.401836,0.000120,0.204775,6331.406250,82807.512154,3.589568,2018.715668,6.506848,4244.888062,0.003456,49.616032,108908.937500
+339,1.374082,1.401836,0.000120,0.166193,6327.891113,82853.511638,3.426944,2020.361206,6.509600,4239.675049,0.002976,49.808254,108908.937500
+340,1.359741,1.401836,0.000119,0.159000,6308.205566,83112.066416,3.356256,2018.082993,6.494976,4222.701538,0.003520,49.449120,108908.937500
+341,1.341903,1.401836,0.000118,0.166805,6317.018555,82996.115250,3.486496,2018.304276,6.518112,4231.154236,0.003648,49.431679,108908.937500
+342,1.395497,1.401836,0.000118,0.188705,6321.744629,82934.068169,3.480800,2018.450897,6.500032,4235.466248,0.003648,49.774239,108908.937500
+343,1.352996,1.401836,0.000117,0.186749,6379.217285,82186.885407,3.481824,2087.512283,6.513856,4224.354675,0.003360,49.310848,108908.937500
+344,1.316176,1.401836,0.000117,0.169106,6314.944824,83023.369894,3.426624,2028.371460,6.505504,4219.099060,0.003456,49.447552,108908.937500
+345,1.303779,1.401836,0.000116,0.159745,6357.660645,82465.552868,3.465824,2081.039627,6.535968,4209.021423,0.003392,49.555328,108908.937500
+346,1.338077,1.401836,0.000116,0.160044,6359.479980,82441.960917,3.447968,2036.460251,6.491520,4255.363953,0.003712,49.613377,108908.937500
+347,1.341638,1.401836,0.000115,0.169222,6309.568848,83094.108751,3.404320,2022.481339,6.525792,4219.242737,0.003584,49.804161,108908.937500
+348,1.359211,1.401836,0.000114,0.205495,6296.755371,83263.199712,3.428928,2014.682083,6.491616,4214.594788,0.003776,49.558784,108908.937500
+349,1.292385,1.401836,0.000114,0.151000,6318.837402,82972.225208,3.482304,2021.280457,6.516896,4229.796814,0.003168,49.664417,108908.937500
+350,1.336500,1.401836,0.000113,0.189541,6320.989258,82943.978959,3.465728,2017.458298,6.818208,4235.489258,0.003456,49.555840,108908.937500
+351,1.360193,1.401836,0.000113,0.268535,6343.214355,82653.363203,3.328992,2021.161072,6.508224,4254.520691,0.003136,49.612896,108908.937500
+352,1.404982,1.401836,0.000112,0.170323,6317.104980,82994.979761,3.491616,2013.434174,6.485472,4236.156311,0.003424,49.441566,108908.937500
+353,1.359454,1.401836,0.000112,0.157117,6331.236328,82809.734597,3.307968,2018.623810,6.515904,4245.041077,0.003840,49.650497,108908.937500
+354,1.312837,1.401836,0.000111,0.179731,6320.554688,82949.681780,3.360064,2023.049454,6.519808,4229.636719,0.003872,49.773567,108908.937500
+355,1.388402,1.401836,0.000111,0.195341,6316.641113,83001.074558,3.536992,2025.839249,6.497472,4222.533691,0.003232,50.073921,108908.937500
+356,1.352636,1.401836,0.000110,0.146554,6346.951172,82604.700399,3.448032,2046.474899,6.512544,4233.061096,0.003712,49.421310,108908.937500
+357,1.382410,1.401836,0.000110,0.210430,6311.306152,83071.235548,3.413440,2021.774933,6.513536,4221.561584,0.003904,49.876926,108908.937500
+358,1.387089,1.401836,0.000110,0.209125,6370.646973,82297.449890,3.543968,2062.645386,6.481664,4240.456238,0.003648,49.312607,108908.937500
+359,1.257791,1.401836,0.000109,0.153765,6305.100098,83153.001837,3.510464,2016.546097,6.490208,4220.960938,0.003776,49.555775,108908.937500
+360,1.345636,1.401836,0.000109,0.157162,6314.994629,83022.715110,3.421184,2018.662033,6.496096,4228.774658,0.003808,49.603039,108908.937500
+361,1.377840,1.401836,0.000108,0.186474,6319.934570,82957.820871,3.416544,2012.355881,6.526592,4240.146362,0.003744,49.431904,108908.937500
+362,1.333666,1.401836,0.000108,0.179946,6323.625488,82909.400781,3.458112,2025.966293,6.508448,4229.734253,0.003552,49.857246,108908.937500
+363,1.356241,1.401836,0.000107,0.162459,6311.403809,83069.950189,3.468704,2020.650116,6.497728,4223.159119,0.003840,49.556992,108908.937500
+364,1.331657,1.401836,0.000107,0.166414,6383.345215,82133.737461,3.442112,2086.818771,6.485248,4229.002686,0.003808,49.538624,108908.937500
+365,1.368689,1.401836,0.000107,0.167335,6375.087891,82240.121077,3.929408,2081.152618,6.712544,4225.832642,0.003328,49.381439,108908.937500
+366,1.353878,1.401836,0.000106,0.171285,6331.716309,82803.457143,3.392736,2020.412796,6.515392,4243.774963,0.003264,49.563774,108908.937500
+367,1.345177,1.401836,0.000106,0.148793,6323.097656,82916.321794,3.488288,2020.659821,6.504192,4234.682617,0.003296,49.662815,108908.937500
+368,1.354756,1.401836,0.000106,0.160956,6328.715820,82842.714839,3.408736,2018.167267,6.514560,4242.303772,0.003360,50.281090,108908.937500
+369,1.360016,1.401836,0.000105,0.168634,6310.303223,83084.438497,3.415712,2014.609528,6.499744,4227.165100,0.003712,49.504002,108908.937500
+370,1.289404,1.401836,0.000105,0.134494,6318.589844,82975.476010,3.516224,2020.803909,6.490720,4230.086243,0.003712,49.666721,108908.937500
+371,1.349306,1.401836,0.000105,0.181257,6302.312012,83189.787974,3.434016,2016.436584,6.500928,4218.285217,0.003776,49.590687,108908.937500
+372,1.373928,1.401836,0.000104,0.144088,6328.482910,82845.763739,3.492640,2023.318222,6.495200,4237.276306,0.003776,49.852898,108908.937500
+373,1.312651,1.401836,0.000104,0.161136,6318.580078,82975.604252,3.423968,2014.663239,6.535296,4236.513184,0.003712,49.360928,108908.937500
+374,1.321823,1.401836,0.000104,0.150005,6298.855957,83235.432526,3.497056,2020.578354,6.483360,4210.537109,0.003712,49.668159,108908.937500
+375,1.378791,1.401836,0.000103,0.180145,6300.058594,83219.543469,3.394048,2019.265808,6.488640,4213.316711,0.003680,49.561153,108908.937500
+376,1.366055,1.401836,0.000103,0.141854,6313.937500,83036.615424,3.320096,2024.376572,6.504096,4222.066895,0.003360,49.591358,108908.937500
+377,1.307925,1.401836,0.000103,0.138054,6312.740723,83052.357610,3.438176,2013.836380,6.527232,4231.585815,0.003488,49.289856,108908.937500
+378,1.324249,1.401836,0.000103,0.150530,6303.692383,83171.571225,3.451840,2013.421646,6.524032,4222.596313,0.003584,49.608448,108908.937500
+379,1.305159,1.401836,0.000102,0.134413,6298.562500,83239.310557,3.381088,2011.305908,6.495552,4219.749390,0.003552,49.551392,108908.937500
+380,1.379794,1.401836,0.000102,0.186374,6368.750977,82321.950086,3.439712,2057.566467,6.504576,4242.741638,0.003680,50.429855,108908.937500
+381,1.369458,1.401836,0.000102,0.166851,6319.319824,82965.891043,3.331424,2029.418503,6.499584,4222.403992,0.003840,49.596863,108908.937500
+382,1.343205,1.401836,0.000102,0.139009,6361.764160,82412.360283,3.462368,2065.785721,6.482400,4228.439270,0.003808,49.514591,108908.937500
+383,1.285721,1.401836,0.000102,0.147778,6328.326172,82847.815640,3.406400,2023.951279,6.528384,4236.798767,0.003488,49.437153,108908.937500
+384,1.380865,1.401836,0.000101,0.162048,6339.782227,82698.108746,3.398496,2026.364197,6.518368,4245.825500,0.003520,49.643425,108908.937500
+385,1.351010,1.401836,0.000101,0.183405,6330.004395,82825.850872,3.377696,2022.605621,6.509920,4239.837830,0.003456,49.538784,108908.937500
+386,1.366956,1.401836,0.000101,0.151915,6317.083984,82995.255611,3.473696,2015.585144,6.801024,4233.890686,0.003776,49.258530,108908.937500
+387,1.382017,1.401836,0.000101,0.178531,6327.277344,82861.548738,3.370464,2015.138245,6.524096,4244.495972,0.003328,49.644608,108908.937500
+388,1.369122,1.401836,0.000101,0.168100,6335.352051,82755.937760,3.430848,2022.521469,6.503136,4245.228333,0.003168,49.626465,108908.937500
+389,1.404818,1.401836,0.000101,0.169657,6303.495117,83174.174050,3.432864,2009.766434,6.526880,4226.188477,0.003168,49.473248,108908.937500
+390,1.235723,1.401836,0.000101,0.132987,6315.982422,83009.730709,3.475008,2037.401474,6.493312,4210.393250,0.003168,50.089119,108908.937500
+391,1.311336,1.401836,0.000100,0.230595,6323.518066,82910.809220,3.381408,2017.961670,6.502432,4238.148865,0.003744,49.460320,108908.937500
+392,1.318072,1.401836,0.000100,0.157000,6337.111328,82732.963468,3.368224,2022.593964,6.492160,4247.133484,0.003680,49.446529,108908.937500
+393,1.372988,1.401836,0.000100,0.153285,6303.337402,83176.255138,3.358176,2014.693375,6.500192,4220.872131,0.003456,49.871265,108908.937500
+394,1.328925,1.401836,0.000100,0.180743,6306.251465,83137.820133,3.417312,2016.162140,6.500672,4222.570862,0.003296,49.560162,108908.937500
+395,1.373340,1.401836,0.000100,0.195646,6340.860840,82684.041370,3.369376,2024.470291,6.489792,4248.816711,0.003392,49.656319,108908.937500
+396,1.351218,1.401836,0.000100,0.156720,6294.054688,83298.926690,3.537472,2015.801651,6.510912,4210.311035,0.003808,49.789310,108908.937500
+397,1.324154,1.401836,0.000100,0.139759,6327.396484,82859.988511,3.472800,2017.604797,6.511680,4242.299133,0.003200,49.477665,108908.937500
+398,1.333024,1.401836,0.000100,0.160029,6354.762207,82503.165802,3.387616,2051.304398,6.510016,4235.383240,0.003840,50.125473,108908.937500
+399,1.349215,1.337912,0.000100,0.150554,11404.865234,45970.556357,13.171680,2016.942093,6.514528,4219.942932,0.003904,49.350529,108908.937500

logs/in2_train_csv/CP_Training_log12_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 100
+  run: 12
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log13.csv

@@ -0,0 +1,323 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,1.719764,1.683354,0.000003,0.917699,15904.565430,32964.622788,3.017888,4236.996201,6.515424,4570.965271,0.003232,62.618080,107212.937500
+1,1.689209,1.683354,0.000006,0.980872,8602.978516,60942.614125,3.486848,4004.690201,6.368896,4530.903809,0.003808,49.570721,108908.937500
+2,1.710424,1.683354,0.000009,1.231633,8313.789062,63062.461178,4.534016,4090.546906,6.342912,4154.729523,0.003168,49.533024,108908.937500
+3,1.731818,1.683354,0.000012,0.900236,6208.146973,84451.608879,3.515936,1990.539001,6.357024,4150.451294,0.003168,49.365250,108908.937500
+4,1.702109,1.683354,0.000015,0.943458,6212.172363,84396.885556,3.405504,1991.149872,6.380128,4153.755157,0.003456,49.508450,108908.937500
+5,1.766153,1.683354,0.000018,0.829794,6250.523438,83879.055129,3.395328,2008.096100,6.367968,4175.291229,0.003648,49.371586,108908.937500
+6,1.703925,1.683354,0.000021,0.908286,6372.131348,82278.278867,3.343328,2084.362701,6.385152,4220.766418,0.002880,49.271137,108908.937500
+7,1.660973,1.683354,0.000024,0.785771,6383.071777,82137.255899,3.331904,2029.901703,6.393440,4285.983337,0.003008,49.563042,108908.937500
+8,1.628360,1.683354,0.000027,0.738050,6420.096191,81663.573936,3.373664,2036.583923,6.389440,4316.306519,0.003264,49.451359,108908.937500
+9,1.665800,1.683354,0.000030,0.737277,6401.355957,81902.647426,3.458144,2035.000381,6.388384,4299.074097,0.002848,49.457249,108908.937500
+10,1.610076,1.683354,0.000033,0.866796,6412.789062,81756.626468,3.444320,2034.575897,6.396960,4310.911804,0.003328,49.487137,108908.937500
+11,1.618258,1.683354,0.000036,0.716137,6374.736816,82244.650266,3.589536,2030.329361,6.398400,4277.050598,0.003712,49.414528,108908.937500
+12,1.605687,1.683354,0.000039,0.753749,6461.937500,81134.798967,3.450176,2101.020203,6.405984,4293.612854,0.003456,49.476894,108908.937500
+13,1.612419,1.683354,0.000042,0.824020,6356.307617,82483.106793,3.518688,2021.296432,6.386976,4267.435486,0.010240,49.695168,108908.937500
+14,1.638093,1.683354,0.000045,0.825805,6346.771973,82607.032718,3.515360,2020.449387,6.380128,4258.522766,0.003552,49.951584,108908.937500
+15,1.627281,1.683354,0.000048,0.853864,6335.848145,82749.458011,3.471424,2022.067444,6.379904,4246.407349,0.003456,49.589088,108908.937500
+16,1.602286,1.683354,0.000051,0.850724,6329.666504,82830.272286,3.547168,2024.266037,6.378624,4238.197083,0.003264,49.347679,108908.937500
+17,1.543830,1.683354,0.000054,0.725071,6331.238281,82809.709051,3.502912,2029.177994,6.392384,4234.731628,0.003104,49.445568,108908.937500
+18,1.574815,1.683354,0.000057,0.681338,6321.410156,82938.456300,3.524704,2008.892929,6.404576,4245.269104,0.003584,49.350880,108908.937500
+19,1.587047,1.683354,0.000060,0.706018,6342.468750,82663.079735,3.440000,2014.729004,6.397120,4260.525146,0.003264,49.426369,108908.937500
+20,1.558500,1.683354,0.000063,0.864713,6334.121582,82772.013958,3.411456,2016.201279,6.400448,4250.666016,0.002944,49.436672,108908.937500
+21,1.628016,1.683354,0.000066,0.875894,6310.748047,83078.582144,3.440672,2009.129807,6.423840,4234.434631,0.003488,49.354015,108908.937500
+22,1.556988,1.683354,0.000069,0.796957,6326.022461,82877.985849,3.457472,2009.259720,6.372128,4249.452942,0.003680,49.504318,108908.937500
+23,1.555099,1.683354,0.000072,0.853955,6324.018555,82904.247587,3.514560,2013.542511,6.383616,4243.186890,0.003904,49.405342,108908.937500
+24,1.524715,1.683354,0.000075,0.610126,6327.998535,82852.105146,3.427648,2036.324112,6.413184,4224.467285,0.003360,49.422241,108908.937500
+25,1.595337,1.683354,0.000078,0.675492,6358.189453,82458.694234,3.480192,2036.230606,6.382240,4254.767883,0.002752,49.358238,108908.937500
+26,1.504139,1.683354,0.000081,0.909033,6344.166504,82640.958379,3.469472,2056.723679,6.406272,4220.221252,0.003104,49.388447,108908.937500
+27,1.509489,1.683354,0.000084,0.616056,6334.266113,82770.125319,3.459520,2021.744614,6.384704,4244.893250,0.003648,49.843166,108908.937500
+28,1.541055,1.683354,0.000087,0.737969,6335.221191,82757.647154,3.442624,2021.190628,6.385088,4246.898926,0.003168,49.363617,108908.937500
+29,1.520158,1.683354,0.000090,0.638712,6310.052734,83087.736675,3.461792,2014.485245,6.382560,4228.257996,0.003904,49.533600,108908.937500
+30,1.497158,1.683354,0.000093,0.621202,6331.833008,82801.931029,3.460192,2019.842682,6.386496,4244.810425,0.003872,49.395073,108908.937500
+31,1.558612,1.683354,0.000096,0.627915,6387.333984,82082.446492,3.482176,2064.046051,6.389120,4256.126587,0.003616,49.356770,108908.937500
+32,1.469230,1.683354,0.000099,0.677005,6318.882812,82971.628935,3.470112,2014.762299,6.382560,4236.726990,0.003232,49.584225,108908.937500
+33,1.531537,1.683354,0.000102,0.695561,6318.933105,82970.968556,3.369088,2017.360825,6.382496,4234.590149,0.003072,49.276958,108908.937500
+34,1.509960,1.683354,0.000105,1.246444,6330.868652,82814.543910,3.439040,2013.222931,6.386240,4250.474304,0.003200,49.375137,108908.937500
+35,1.521332,1.683354,0.000108,0.660045,6320.122070,82955.359749,3.392256,2014.622009,6.387744,4238.392944,0.003616,49.394337,108908.937500
+36,1.499588,1.683354,0.000111,0.790585,6328.807129,82841.519629,3.440608,2018.627228,6.371584,4243.012268,0.003264,49.433823,108908.937500
+37,1.499973,1.683354,0.000114,0.659037,6330.887207,82814.301196,3.392256,2018.592758,6.401568,4245.113831,0.003488,49.430050,108908.937500
+38,1.491481,1.683354,0.000117,0.872585,6324.114258,82902.992993,3.512768,2010.380875,6.364096,4246.420288,0.003328,49.499935,108908.937500
+39,1.515249,1.683354,0.000120,0.633788,6316.149414,83007.536021,3.457120,2014.110397,6.374240,4234.705078,0.003872,49.350079,108908.937500
+40,1.475689,1.683354,0.000123,0.798085,6356.197754,82484.532467,3.465600,2020.735229,6.682816,4267.971375,0.003584,49.394112,108908.937500
+41,1.453508,1.683354,0.000126,0.644359,6302.201172,83191.251073,3.412416,2008.398682,6.373056,4226.570190,0.003136,49.424671,108908.937500
+42,1.502223,1.683354,0.000129,0.645058,6360.252441,82431.948233,3.401824,2054.057938,6.389632,4239.113892,0.003392,49.326496,108908.937500
+43,1.504963,1.683354,0.000132,0.666824,6317.300781,82992.407383,3.527104,2015.292786,6.376608,4234.756897,0.003520,49.375679,108908.937500
+44,1.425290,1.683354,0.000135,1.220775,6359.732910,82438.682160,3.518176,2050.697968,6.400352,4241.546875,0.003456,49.599361,108908.937500
+45,1.504238,1.683354,0.000138,0.619391,6322.991699,82917.711258,3.526144,2016.472687,6.380512,4238.870361,0.003488,49.769535,108908.937500
+46,1.491822,1.683354,0.000141,0.657558,6328.943848,82839.730075,3.397216,2018.656052,6.435136,4243.119446,0.003584,49.373665,108908.937500
+47,1.515073,1.683354,0.000144,0.795129,6324.231445,82901.456807,3.477024,2010.298843,6.397760,4246.724854,0.003776,49.349888,108908.937500
+48,1.509569,1.683354,0.000147,0.972179,6319.159180,82968.000187,3.483712,2021.580795,6.380640,4230.384888,0.003424,49.375935,108908.937500
+49,1.487431,1.683354,0.000150,0.692380,6332.062500,82798.930048,3.511264,2020.505600,6.650784,4243.835388,0.003104,49.559040,108908.937500
+50,1.519441,1.683354,0.000153,0.609859,6325.658203,82882.758310,3.581440,2024.942719,6.391616,4233.059753,0.003456,49.728127,108908.937500
+51,1.539274,1.683354,0.000156,0.679308,6338.995117,82708.377323,3.560320,2011.280609,6.384960,4259.961975,0.003616,49.869698,108908.937500
+52,1.548381,1.683354,0.000159,0.592827,6296.164551,83271.012975,3.508992,2011.416214,6.372288,4217.687256,0.003488,49.251999,108908.937500
+53,1.472854,1.683354,0.000162,0.874715,6324.349609,82899.907877,3.516384,2009.700134,6.615456,4247.132385,0.003680,49.434238,108908.937500
+54,1.479821,1.683354,0.000165,0.772366,6332.937012,82787.496391,3.472608,2018.252625,6.384288,4247.409485,0.003008,49.466496,108908.937500
+55,1.559876,1.683354,0.000168,0.672268,6319.433594,82964.397398,3.385440,2010.317047,6.389184,4241.966797,0.003872,49.431873,108908.937500
+56,1.525480,1.683354,0.000171,0.687398,6329.410156,82833.626998,3.565312,2019.058243,6.399392,4241.850342,0.003008,50.568993,108908.937500
+57,1.459871,1.683354,0.000174,0.667996,6321.778809,82933.619773,3.489472,2011.045135,6.392320,4243.307739,0.002944,49.572609,108908.937500
+58,1.471684,1.683354,0.000177,0.793316,6347.725586,82594.622736,3.443680,2032.030731,6.678208,4248.222168,0.003296,49.395649,108908.937500
+59,1.480517,1.683354,0.000180,0.733348,6323.140625,82915.758338,3.521760,2015.585373,6.385248,4240.325562,0.003392,49.390976,108908.937500
+60,1.519410,1.683354,0.000183,0.731512,6331.638672,82804.472455,3.544672,2021.187195,6.394912,4243.270996,0.003264,49.271935,108908.937500
+61,1.455516,1.683354,0.000186,0.693638,6323.941406,82905.258971,3.547168,2014.361694,6.382048,4242.354431,0.002848,49.344894,108908.937500
+62,1.486469,1.683354,0.000189,0.782711,6324.214844,82901.674430,3.453696,2013.077621,6.380960,4243.979004,0.003456,49.374592,108908.937500
+63,1.483447,1.683354,0.000192,0.891996,6335.917969,82748.546081,3.496640,2020.975922,6.370656,4247.582520,0.003424,49.559681,108908.937500
+64,1.452058,1.683354,0.000195,0.836276,6364.759766,82373.572500,3.495168,2062.025497,6.392032,4235.449524,0.003392,49.428032,108908.937500
+65,1.500889,1.683354,0.000198,0.840384,6336.292969,82743.648784,3.414496,2023.047226,6.518144,4245.929321,0.003168,49.410721,108908.937500
+66,1.483080,1.683354,0.000201,0.778910,6334.956055,82761.110807,3.505984,2017.556671,6.383232,4250.141357,0.003200,49.383362,108908.937500
+67,1.524449,1.683354,0.000204,0.887582,6310.816895,83077.675800,3.529408,2011.570953,6.372032,4232.040222,0.003744,49.372448,108908.937500
+68,1.493887,1.683354,0.000207,0.679346,6330.847656,82814.818563,3.500960,2018.125931,6.376960,4244.622070,0.003456,50.267967,108908.937500
+69,1.532737,1.683354,0.000210,0.713714,6315.896973,83010.853766,3.532992,2019.757889,6.376896,4228.628906,0.003616,49.656063,108908.937500
+70,1.498729,1.683354,0.000213,0.850240,6321.528809,82936.899582,3.477920,2015.558640,6.388384,4238.616882,0.003392,49.499138,108908.937500
+71,1.501005,1.683354,0.000216,0.656128,6370.353027,82301.247317,3.472512,2075.738037,6.394496,4227.272766,0.003136,49.484352,108908.937500
+72,1.435198,1.683354,0.000219,0.881135,6322.629395,82922.462679,3.479776,2016.386398,6.373472,4238.955444,0.003264,49.482018,108908.937500
+73,1.507871,1.683354,0.000222,0.874623,6320.861328,82945.657685,3.484832,2010.013397,6.374144,4243.610657,0.002912,49.360863,108908.937500
+74,1.489377,1.683354,0.000225,0.867319,6323.052734,82916.910870,3.495232,2014.255096,6.384512,4241.449036,0.003360,49.529568,108908.937500
+75,1.502547,1.683354,0.000228,0.743009,6332.577148,82792.200981,3.525536,2015.218475,6.381344,4250.080566,0.003232,49.371231,108908.937500
+76,1.455447,1.683354,0.000231,0.777229,6310.788086,83078.055048,3.510784,2011.067337,6.376352,4232.434753,0.003392,49.444641,108908.937500
+77,1.513057,1.683354,0.000234,0.698188,6349.456055,82572.112553,3.566368,2030.187515,6.387008,4251.981323,0.003232,49.343327,108908.937500
+78,1.487326,1.683354,0.000237,0.746300,6344.279785,82639.482771,3.592448,2046.403488,6.375168,4230.146301,0.003360,49.820160,108908.937500
+79,1.497366,1.683354,0.000240,0.797108,6336.672363,82738.694687,3.464256,2010.920288,6.586624,4258.209229,0.003296,49.512417,108908.937500
+80,1.496414,1.683354,0.000243,0.684250,6305.024902,83153.993540,3.424864,2012.448166,6.375424,4224.952515,0.003552,49.882847,108908.937500
+81,1.544732,1.683354,0.000246,0.733620,6309.066406,83100.726199,3.442464,2023.080780,6.392640,4218.869019,0.003104,49.330208,108908.937500
+82,1.565590,1.683354,0.000249,0.782071,6314.537598,83028.724098,3.441088,2020.186722,6.374912,4227.297302,0.003584,49.306110,108908.937500
+83,1.488052,1.683354,0.000252,0.750319,6320.021973,82956.673611,3.554208,2012.422287,6.398016,4240.296631,0.003712,49.383362,108908.937500
+84,1.506120,1.683354,0.000255,0.825944,6357.747070,82464.431850,3.529024,2063.077881,6.369632,4227.474121,0.003072,49.328609,108908.937500
+85,1.508562,1.683354,0.000258,0.808001,6323.588379,82909.887327,3.560800,2016.383514,6.377952,4239.997192,0.003072,49.327648,108908.937500
+86,1.506531,1.683354,0.000261,0.845452,6321.491211,82937.392856,3.424288,2012.783524,6.387552,4241.135498,0.003392,49.865250,108908.937500
+87,1.568609,1.683354,0.000264,0.838475,6346.159668,82615.002999,3.437568,2055.654861,6.385600,4223.200195,0.003264,49.555008,108908.937500
+88,1.578616,1.683354,0.000267,0.843208,6316.879883,82997.937230,3.435104,2016.792068,6.370432,4232.907593,0.003744,49.438145,108908.937500
+89,1.520792,1.683354,0.000270,0.799951,6328.621094,82843.954826,3.658304,1999.598694,6.379680,4261.590698,0.003008,49.438946,108908.937500
+90,1.589205,1.683354,0.000273,0.781977,6301.027832,83206.742452,3.488096,2015.294312,6.370336,4218.176575,0.003392,49.709793,108908.937500
+91,1.535868,1.683354,0.000276,0.853053,6363.633789,82388.147618,3.532896,2066.524734,6.383808,4229.526611,0.003072,49.741665,108908.937500
+92,1.544926,1.683354,0.000279,0.778064,6364.683594,82374.558339,3.453504,2043.298370,6.394496,4254.082397,0.003264,49.487297,108908.937500
+93,1.568713,1.683354,0.000282,0.732620,6315.086914,83021.501863,3.417504,2012.716110,6.373056,4235.131165,0.003744,49.511040,108908.937500
+94,1.447728,1.683354,0.000285,0.837362,6328.150391,82850.116959,3.451808,2018.844696,6.396992,4241.057922,0.003360,50.017952,108908.937500
+95,1.528693,1.683354,0.000288,0.767508,6318.460938,82977.168837,3.362368,2017.057892,6.397440,4234.214600,0.003744,49.440514,108908.937500
+96,1.526013,1.683354,0.000291,0.740605,6334.853516,82762.450419,3.431264,2018.016708,6.381280,4249.811768,0.003520,49.251457,108908.937500
+97,1.522310,1.683354,0.000294,0.726241,6366.418457,82352.111087,3.518272,2054.183365,6.398720,4244.901062,0.003264,49.436192,108908.937500
+98,1.517379,1.683354,0.000297,0.750487,6335.560059,82753.220734,3.514144,2027.682617,6.373312,4240.735229,0.003200,49.270496,108908.937500
+99,1.542074,1.683354,0.000300,0.946765,6316.633789,83001.170799,3.427776,2015.213348,6.397312,4234.244873,0.003904,49.384895,108908.937500
+100,1.560971,1.683354,0.000300,0.828052,6193.538086,84650.807459,3.350720,1986.990509,6.356000,4139.365906,0.003136,49.399872,108908.937500
+101,1.590868,1.683354,0.000300,0.785824,6223.603516,84241.870274,3.345792,1992.218323,6.363008,4164.114502,0.003104,49.622913,108908.937500
+102,1.500099,1.683354,0.000300,0.774049,6261.979980,83725.595041,3.370688,1995.605331,6.354464,4199.357971,0.003456,49.329632,108908.937500
+103,1.553445,1.683354,0.000300,0.751207,6281.820801,83461.151890,3.271712,2016.959991,6.367808,4197.821198,0.003712,49.421505,108908.937500
+104,1.622695,1.683354,0.000300,0.952282,6280.207520,83482.591677,3.428512,2006.258682,6.376832,4206.821228,0.003200,49.315647,108908.937500
+105,1.534858,1.683354,0.000300,0.849580,6321.540039,82936.752241,3.439904,2015.264572,6.372992,4239.040405,0.003360,49.402176,108908.937500
+106,1.540255,1.683354,0.000300,0.953627,6364.299805,82379.525806,3.315680,2052.579590,6.396928,4244.675110,0.003616,49.389023,108908.937500
+107,1.606415,1.683354,0.000300,0.749069,6333.168945,82784.464543,3.459136,2016.506760,6.380544,4249.413574,0.003744,49.450562,108908.937500
+108,1.496417,1.683354,0.000300,0.923751,6313.805664,83038.349277,3.287520,2012.084915,6.382848,4234.729431,0.003296,49.342655,108908.937500
+109,1.536228,1.683354,0.000300,0.712602,6318.936523,82970.923676,3.427168,2017.550354,6.387328,4233.713074,0.003360,49.885345,108908.937500
+110,1.559572,1.683354,0.000299,0.844266,6326.958984,82865.718159,3.323008,2019.819427,6.383392,4239.656250,0.003520,49.826080,108908.937500
+111,1.541899,1.683354,0.000299,0.748256,6321.799316,82933.350738,3.363808,2009.344009,6.645856,4245.235291,0.003648,49.235329,108908.937500
+112,1.518180,1.683354,0.000299,0.907726,6326.172852,82876.015610,3.445632,2020.623413,6.385152,4238.038757,0.003232,49.680546,108908.937500
+113,1.504054,1.683354,0.000299,0.855981,6305.272949,83150.722296,3.434784,2015.258224,6.384608,4222.961243,0.003648,49.254559,108908.937500
+114,1.536195,1.683354,0.000299,0.651210,6374.010742,82254.018891,3.461248,2062.077225,6.375264,4244.806213,0.003328,49.305473,108908.937500
+115,1.523702,1.683354,0.000299,0.722316,6380.525391,82170.035836,3.328320,2062.411957,6.395168,4250.826660,0.003616,49.586624,108908.937500
+116,1.471415,1.683354,0.000299,0.910511,6321.893066,82932.120884,3.416224,2014.788940,6.366944,4239.031433,0.003456,50.355042,108908.937500
+117,1.486171,1.683354,0.000298,0.812047,6331.324707,82808.578656,3.369856,2022.284958,6.381632,4241.782104,0.002784,49.517857,108908.937500
+118,1.551453,1.683354,0.000298,0.765060,6317.548340,82989.155254,3.375328,2016.718536,6.500160,4233.645691,0.003872,49.352032,108908.937500
+119,1.563141,1.683354,0.000298,0.746481,6339.210938,82705.561492,3.326240,2017.471588,6.382720,4254.648132,0.003680,49.398594,108908.937500
+120,1.506313,1.683354,0.000298,0.629411,6324.937988,82892.196093,3.341088,2018.381683,6.354688,4239.507385,0.003424,49.384544,108908.937500
+121,1.531366,1.683354,0.000298,0.751968,6304.785156,83157.155558,3.372192,2019.384277,6.382656,4218.294556,0.003552,49.375935,108908.937500
+122,1.583121,1.683354,0.000297,0.722242,6373.510254,82260.477996,3.323168,2064.855377,6.372032,4241.538452,0.003104,49.458977,108908.937500
+123,1.549728,1.683354,0.000297,0.693229,6385.673340,82103.792677,3.418112,2075.536575,6.408928,4242.953796,0.003072,49.388447,108908.937500
+124,1.571770,1.683354,0.000297,0.714164,6307.751953,83118.043305,3.388160,2008.911926,6.384416,4231.774475,0.003296,49.348064,108908.937500
+125,1.530097,1.683354,0.000297,0.595854,6302.296387,83189.994223,3.366144,2014.827560,6.386496,4220.361572,0.002816,49.401890,108908.937500
+126,1.548526,1.683354,0.000296,0.679890,6342.526367,82662.328802,3.353792,2019.102722,6.374976,4256.468445,0.003584,49.319138,108908.937500
+127,1.587535,1.683354,0.000296,0.590154,6309.104980,83100.218117,3.381408,2015.890533,6.382880,4225.912048,0.003936,49.585758,108908.937500
+128,1.606323,1.683354,0.000296,0.611164,6359.267090,82444.720845,3.433600,2067.147781,6.389696,4225.080811,0.003232,49.273502,108908.937500
+129,1.556684,1.683354,0.000295,0.611280,6341.035156,82681.768368,3.402656,2057.618225,6.383840,4216.433594,0.003168,49.299423,108908.937500
+130,1.676277,1.683354,0.000295,0.701634,6325.641113,82882.982232,3.366560,2015.003204,6.394752,4242.654541,0.003488,49.436031,108908.937500
+131,1.559720,1.683354,0.000295,0.634442,6334.891602,82761.952844,3.284800,2019.906418,6.393760,4247.937317,0.003136,49.383938,108908.937500
+132,1.561504,1.683354,0.000294,0.685133,6315.482422,83016.302632,15.352640,2015.823288,6.370240,4220.279724,0.003232,49.686401,108908.937500
+133,1.544661,1.683354,0.000294,0.579670,6326.395996,82873.092409,3.262336,2020.009918,6.377408,4239.351807,0.003200,49.438305,108908.937500
+134,1.552186,1.683354,0.000294,0.637175,6314.066895,83034.913750,3.426816,2019.403641,6.373728,4227.613464,0.003552,49.272705,108908.937500
+135,1.576610,1.683354,0.000293,0.587117,6309.438477,83095.825714,3.407008,2016.359055,6.364800,4225.906372,0.003680,49.421345,108908.937500
+136,1.529440,1.683354,0.000293,0.604033,6360.191895,82432.732957,3.296064,2091.072510,6.368064,4202.109711,0.003616,49.365601,108908.937500
+137,1.524042,1.683354,0.000293,0.593304,6319.366211,82965.282039,3.341216,2019.424393,6.391936,4231.817322,0.003072,49.414272,108908.937500
+138,1.582924,1.683354,0.000292,0.546919,6309.434570,83095.877159,3.385536,2017.964554,6.375488,4224.373657,0.003936,49.378433,108908.937500
+139,1.578228,1.683354,0.000292,0.770910,6318.262695,82979.772333,3.394016,2007.354858,6.608480,4243.598511,0.003680,49.398750,108908.937500
+140,1.595851,1.683354,0.000291,0.580508,6296.668945,83264.342552,3.507776,2014.073914,6.364096,4214.204773,0.003456,50.400063,108908.937500
+141,1.587405,1.683354,0.000291,0.607911,6355.135254,82498.322861,3.423904,2052.595322,6.370144,4235.109558,0.003296,49.703522,108908.937500
+142,1.570241,1.683354,0.000290,0.560675,6316.685059,83000.497118,3.346240,2013.230026,6.379520,4236.267822,0.003648,49.536320,108908.937500
+143,1.514309,1.683354,0.000290,1.853380,6304.187012,83165.045552,3.342016,2014.788284,6.367712,4222.233459,0.003680,49.526592,108908.937500
+144,1.589352,1.683354,0.000290,0.571877,6310.077148,83087.415204,3.377568,2014.479828,6.625632,4227.934387,0.002784,49.674686,108908.937500
+145,1.502340,1.683354,0.000289,0.564554,6301.995117,83193.971155,3.327744,2008.611862,6.376640,4226.437439,0.003168,49.288734,108908.937500
+146,1.517694,1.683354,0.000289,0.577031,6306.293945,83137.260100,3.389856,2009.645416,6.382592,4229.463989,0.002880,49.459778,108908.937500
+147,1.520943,1.683354,0.000288,0.518964,6292.176758,83323.787646,3.348384,2014.015732,6.362400,4211.175537,0.003456,49.289150,108908.937500
+148,1.562016,1.683354,0.000288,0.604357,6279.468750,83492.413271,3.374112,2011.221695,6.375936,4201.204895,0.003744,49.384930,108908.937500
+149,1.563453,1.683354,0.000287,0.614633,6333.330078,82782.358338,3.390976,2044.602249,6.379744,4221.523438,0.003104,49.464352,108908.937500
+150,1.517942,1.683354,0.000287,0.680002,6318.638672,82974.834806,3.393408,2032.470398,6.389408,4219.142578,0.003104,49.333057,108908.937500
+151,1.538941,1.683354,0.000286,0.514121,6293.206055,83310.159471,3.374016,2011.136032,6.408672,4214.706055,0.003232,49.555424,108908.937500
+152,1.583701,1.683354,0.000286,0.473033,6299.010742,83233.387187,3.376288,2007.488968,6.369696,4224.341736,0.003808,49.486366,108908.937500
+153,1.539420,1.683354,0.000285,0.607666,6288.103027,83377.768736,3.353376,2009.856079,6.369536,4211.173889,0.003456,49.394688,108908.937500
+154,1.568313,1.683354,0.000284,0.716956,6303.739746,83170.946314,3.307552,2015.042969,6.369664,4221.605774,0.003040,49.428898,108908.937500
+155,1.547035,1.683354,0.000284,0.480610,6288.742188,83369.294585,3.329120,2009.183029,6.371168,4212.548706,0.003680,49.388672,108908.937500
+156,1.542251,1.683354,0.000283,0.607877,6310.967773,83075.689628,3.402080,2015.374222,6.369824,4228.493958,0.003392,49.390656,108908.937500
+157,1.593042,1.683354,0.000283,0.501831,6307.177734,83125.610547,3.458112,2011.980148,6.369696,4227.757935,0.003648,49.657856,108908.937500
+158,1.604503,1.683354,0.000282,0.608033,6336.211426,82744.713642,3.385728,2065.825806,6.373344,4203.203430,0.003296,49.452606,108908.937500
+159,1.554489,1.683354,0.000282,0.573337,6309.680664,83092.636207,3.352320,2021.767990,6.381184,4220.805725,0.003744,49.358047,108908.937500
+160,1.589306,1.683354,0.000281,0.474656,6356.365723,82482.352790,3.467168,2075.269180,6.381120,4213.902161,0.003168,49.479298,108908.937500
+161,1.558533,1.683354,0.000280,0.487487,6343.323242,82651.944412,3.336384,2045.239059,6.380096,4230.907898,0.003456,49.468800,108908.937500
+162,1.552307,1.683354,0.000280,0.485688,6305.584473,83146.614287,3.404864,2016.367920,6.383104,4222.052917,0.002976,49.382145,108908.937500
+163,1.561074,1.683354,0.000279,0.431450,6313.937012,83036.621846,3.238272,2015.352783,6.384864,4231.344055,0.003264,49.437504,108908.937500
+164,1.494718,1.683354,0.000278,0.451777,6313.209961,83046.184626,3.426848,2009.973831,6.364416,4236.021973,0.003424,49.458942,108908.937500
+165,1.550173,1.683354,0.000278,0.524928,6329.386230,82833.940118,3.313792,2014.950623,6.386784,4247.275818,0.003488,49.460033,108908.937500
+166,1.549638,1.683354,0.000277,0.518313,6318.250977,82979.926240,3.353376,2019.436005,6.386560,4231.609802,0.003808,49.475201,108908.937500
+167,1.567677,1.683354,0.000276,0.561204,6319.494629,82963.596108,3.341952,2016.579025,6.374112,4235.559875,0.003520,49.682590,108908.937500
+168,1.598396,1.683354,0.000276,0.464795,6299.201660,83230.864526,3.351040,2015.919403,6.367264,4216.159119,0.003008,49.404194,108908.937500
+169,1.517734,1.683354,0.000275,0.512758,6302.750000,83184.006981,3.415296,2011.648117,6.382208,4223.916565,0.003520,49.415680,108908.937500
+170,1.526498,1.683354,0.000274,0.498533,6296.948242,83260.649419,3.365920,2008.009476,6.360512,4221.856201,0.003008,49.341152,108908.937500
+171,1.589458,1.683354,0.000274,0.562132,6331.358887,82808.131616,3.345536,2049.680710,6.366304,4214.486938,0.003872,49.503647,108908.937500
+172,1.551229,1.683354,0.000273,0.486791,6348.522949,82584.248997,3.361952,2061.093231,6.382176,4219.914001,0.003168,49.761311,108908.937500
+173,1.536721,1.683354,0.000272,0.472513,6330.280762,82822.234864,3.429984,2028.647995,6.375648,4234.340820,0.003552,49.498142,108908.937500
+174,1.561139,1.683354,0.000271,0.479635,6308.287109,83110.992082,3.376864,2019.980270,6.363360,4221.128296,0.003520,49.434368,108908.937500
+175,1.546170,1.683354,0.000271,0.508718,6284.762207,83422.090244,3.428672,2010.275467,6.389344,4207.302124,0.003424,49.410080,108908.937500
+176,1.510554,1.683354,0.000270,0.495505,6326.662109,82869.606585,3.419488,2015.469482,6.367968,4244.170959,0.003424,49.309120,108908.937500
+177,1.594345,1.683354,0.000269,0.470225,6303.453125,83174.728138,3.375584,2019.284958,6.392032,4217.055847,0.003296,49.376545,108908.937500
+178,1.565544,1.683354,0.000268,0.654264,6356.008789,82486.984741,3.548960,2067.645172,6.378080,4220.580750,0.003072,49.827839,108908.937500
+179,1.570838,1.683354,0.000268,0.485405,6338.306152,82717.367606,3.369312,2060.576813,6.377824,4210.623474,0.003392,49.372318,108908.937500
+180,1.608467,1.683354,0.000267,0.454251,6348.955078,82578.628065,3.298016,2078.157272,6.376384,4203.790771,0.002944,49.381184,108908.937500
+181,1.518504,1.683354,0.000266,0.403720,6300.045898,83219.711166,3.290624,2012.520859,6.407520,4220.416626,0.003104,49.428959,108908.937500
+182,1.519081,1.683354,0.000265,0.370552,6300.372559,83215.396411,3.461248,2013.368073,6.365888,4219.783447,0.003008,49.472927,108908.937500
+183,1.545064,1.683354,0.000265,0.400680,6298.799805,83236.174550,3.297408,2010.557861,6.377824,4221.234375,0.003808,49.366848,108908.937500
+184,1.546265,1.683354,0.000264,0.449947,6286.921875,83393.433293,3.328128,2014.573639,6.381056,4205.411072,0.003488,49.234402,108908.937500
+185,1.517903,1.683354,0.000263,0.383015,6304.947266,83155.017467,3.331328,2015.661407,6.377088,4221.204346,0.003456,50.410946,108908.937500
+186,1.585939,1.683354,0.000262,0.424482,6312.373047,83057.195148,3.363936,2015.795700,6.399968,4229.192932,0.002976,49.604706,108908.937500
+187,1.557112,1.683354,0.000261,0.431516,6293.001953,83312.861478,3.290496,2002.226425,6.378048,4223.686951,0.003616,49.448128,108908.937500
+188,1.538441,1.683354,0.000260,0.456851,6300.703613,83211.024066,3.448416,2009.841064,6.379264,4223.441223,0.003168,49.600288,108908.937500
+189,1.548894,1.683354,0.000260,0.352548,6327.367676,82860.365774,3.244928,2030.542084,6.373952,4229.853638,0.003360,49.368896,108908.937500
+190,1.522655,1.683354,0.000259,0.421385,6354.265137,82509.619715,3.325760,2063.542648,6.361856,4223.408020,0.003488,49.403648,108908.937500
+191,1.544763,1.683354,0.000258,0.371389,6323.341797,82913.120442,3.433440,2012.466446,6.378304,4243.598999,0.003424,49.490398,108908.937500
+192,1.531644,1.683354,0.000257,0.540623,6287.513672,83385.584089,3.464928,2009.766388,6.372128,4210.590210,0.003328,49.405342,108908.937500
+193,1.527502,1.683354,0.000256,0.415703,6304.969727,83154.721234,3.318304,2018.274841,6.408288,4218.913208,0.003584,50.044159,108908.937500
+194,1.537738,1.683354,0.000255,0.441192,6309.684082,83092.591195,3.467232,2010.895447,6.384576,4231.562195,0.002848,49.389183,108908.937500
+195,1.519402,1.683354,0.000254,0.426687,6281.645020,83463.487410,3.267936,2014.196335,6.386592,4200.358826,0.003680,49.458591,108908.937500
+196,1.552346,1.683354,0.000254,0.916381,6314.606445,83027.818842,3.308928,2009.968582,6.391072,4237.619019,0.003744,49.316895,108908.937500
+197,1.558954,1.683354,0.000253,0.491121,6335.280273,82756.875366,3.376736,2077.857910,6.389568,4189.975281,0.003392,49.713951,108908.937500
+198,1.508947,1.683354,0.000252,0.371437,6359.109863,82446.759259,3.453696,2068.787003,6.383296,4222.931030,0.003296,49.616993,108908.937500
+199,1.520034,1.683354,0.000251,0.376074,6309.289551,83097.787125,3.343168,2007.945160,6.361472,4234.187866,0.003840,49.465378,108908.937500
+200,1.515468,1.683354,0.000250,0.336333,6274.529785,83558.133908,3.271840,2091.931290,6.365120,4115.542389,0.003200,49.394402,108908.937500
+201,1.543592,1.683354,0.000249,0.385330,6197.180664,84601.051417,3.341088,1988.023636,6.354176,4142.051880,0.003488,49.518337,108908.937500
+202,1.518495,1.683354,0.000248,0.370330,6216.373047,84339.854775,3.537600,1992.051575,6.343616,4156.979004,0.002976,49.511040,108908.937500
+203,1.594687,1.683354,0.000247,0.327935,6224.027832,84236.127175,3.434816,1991.040146,6.353120,4165.429810,0.003744,49.822239,108908.937500
+204,1.508122,1.683354,0.000246,0.335039,6244.077637,83965.644008,3.242240,1998.693130,6.340736,4178.513000,0.003584,49.319519,108908.937500
+205,1.481425,1.683354,0.000245,0.486750,6300.964844,83207.574237,3.374880,2004.316238,6.370464,4229.395447,0.003168,49.533630,108908.937500
+206,1.561807,1.683354,0.000244,0.434485,6308.093262,83113.546082,3.537280,2014.538467,6.372512,4226.345825,0.003456,49.330688,108908.937500
+207,1.582822,1.683354,0.000244,0.348183,6330.538086,82818.868299,3.416736,2021.239014,6.389440,4241.880920,0.003296,49.632671,108908.937500
+208,1.465932,1.683354,0.000243,0.364471,6351.699219,82542.951412,3.385088,2027.342087,6.381696,4257.275024,0.003360,49.341248,108908.937500
+209,1.552704,1.683354,0.000242,0.500289,6382.091797,82149.868207,3.521600,2067.922791,6.388928,4246.860107,0.003552,49.482143,108908.937500
+210,1.560924,1.683354,0.000241,0.420376,6343.379395,82651.212767,3.391264,2021.791000,6.400640,4254.499573,0.002912,49.294655,108908.937500
+211,1.577441,1.683354,0.000240,0.277977,6369.290527,82314.976487,3.290496,2023.882767,6.406016,4278.304382,0.003872,49.435966,108908.937500
+212,1.523178,1.683354,0.000239,0.341484,6376.564453,82221.077487,3.409184,2029.355576,6.516320,4279.929321,0.003840,49.369057,108908.937500
+213,1.532332,1.683354,0.000238,0.342764,6336.416504,82742.035609,3.363072,2022.624252,6.386976,4246.609802,0.003232,49.328094,108908.937500
+214,1.526189,1.683354,0.000237,0.438300,6379.649902,82181.312145,3.318784,2044.153244,6.386976,4268.447632,0.003040,49.355072,108908.937500
+215,1.536538,1.683354,0.000236,0.413876,6378.226074,82199.657695,3.380416,2030.409225,6.391808,4280.581726,0.003552,49.459873,108908.937500
+216,1.505207,1.683354,0.000235,0.386820,6372.579102,82272.497782,3.387424,2027.364197,6.398496,4278.092834,0.003264,49.385025,108908.937500
+217,1.519254,1.683354,0.000234,0.292447,6370.758301,82296.011754,3.337824,2027.262344,6.387584,4276.308472,0.003424,49.488255,108908.937500
+218,1.547643,1.683354,0.000233,0.326851,6415.389648,81723.485046,3.422816,2060.334320,6.381536,4287.898193,0.002816,49.396767,108908.937500
+219,1.571966,1.683354,0.000232,0.347060,6380.809082,82166.382548,3.348864,2030.717072,6.392960,4283.041199,0.003584,49.333378,108908.937500
+220,1.509900,1.683354,0.000231,0.297242,6356.959961,82474.642474,3.319840,2023.536621,6.375424,4266.348877,0.003232,49.409569,108908.937500
+221,1.497495,1.683354,0.000230,0.336133,6365.624512,82362.382361,3.384096,2052.171036,6.384320,4246.198120,0.002816,49.519840,108908.937500
+222,1.492427,1.683354,0.000229,0.291803,6375.236328,82238.206243,3.415040,2051.880432,6.653120,4255.886780,0.003136,49.432865,108908.937500
+223,1.519729,1.683354,0.000228,0.326629,6357.229980,82471.139413,3.389344,2018.630493,6.385088,4271.324280,0.003200,49.489857,108908.937500
+224,1.466115,1.683354,0.000227,0.374601,6343.836914,82645.251935,3.375360,2023.929535,6.387360,4252.774231,0.003424,49.397663,108908.937500
+225,1.554846,1.683354,0.000226,0.366232,6337.588379,82726.735890,3.340160,2021.047134,6.434496,4249.357178,0.003520,49.447521,108908.937500
+226,1.487506,1.683354,0.000225,0.401156,6335.978516,82747.755332,3.424832,2021.635803,6.384704,4247.203125,0.002720,49.402912,108908.937500
+227,1.479402,1.683354,0.000224,0.284060,6320.689941,82947.906773,3.361440,2020.263550,6.386176,4233.322266,0.003232,49.373375,108908.937500
+228,1.533440,1.683354,0.000223,0.584058,6387.963867,82074.352783,3.378048,2070.901947,6.379008,4250.076843,0.003264,49.251102,108908.937500
+229,1.506774,1.683354,0.000222,0.289331,6334.241211,82770.450720,3.379296,2019.989838,6.392000,4247.161621,0.003616,49.365120,108908.937500
+230,1.499669,1.683354,0.000221,0.358900,6331.552734,82805.596351,3.460672,2021.380997,6.379552,4242.967773,0.003680,49.406528,108908.937500
+231,1.486993,1.683354,0.000220,0.324916,6305.996582,83141.180491,3.497120,2017.253693,6.390752,4221.450134,0.003520,49.468704,108908.937500
+232,1.574881,1.683354,0.000219,0.319280,6340.373535,82690.396251,3.457856,2021.553741,6.376640,4251.653748,0.003200,49.357216,108908.937500
+233,1.508013,1.683354,0.000218,0.246826,6327.013672,82865.001909,3.295360,2025.688904,6.381280,4234.307800,0.003296,49.347649,108908.937500
+234,1.524792,1.683354,0.000217,0.298971,6327.451660,82859.265967,3.334656,2011.886383,6.363904,4248.502014,0.002944,49.373470,108908.937500
+235,1.561522,1.683354,0.000216,0.242304,6412.856445,81755.767414,3.455328,2092.091827,6.384512,4253.440002,0.003712,49.516640,108908.937500
+236,1.532568,1.683354,0.000215,0.292227,6317.352539,82991.727430,3.327936,2024.750443,6.386048,4225.242126,0.003360,49.675007,108908.937500
+237,1.511153,1.683354,0.000214,0.374216,6312.037109,83061.615595,3.369248,2012.825134,6.366976,4232.145691,0.003552,49.358913,108908.937500
+238,1.461659,1.683354,0.000213,0.323926,6330.508789,82819.251575,3.442240,2024.605515,6.373216,4238.821167,0.003584,49.303040,108908.937500
+239,1.480994,1.683354,0.000211,0.243427,6309.649902,83093.041312,3.354208,2017.868332,6.436192,4224.645264,0.003712,49.372833,108908.937500
+240,1.504617,1.683354,0.000210,0.347179,6399.218262,81930.007472,3.360704,2084.712738,6.391360,4247.030640,0.003168,49.733089,108908.937500
+241,1.484900,1.683354,0.000209,0.235026,6329.705078,82829.767506,3.307520,2021.444717,6.393184,4241.205872,0.003328,49.436958,108908.937500
+242,1.526330,1.683354,0.000208,0.301012,6328.148438,82850.142530,3.355296,2017.778946,6.391840,4243.333862,0.002752,49.302593,108908.937500
+243,1.494656,1.683354,0.000207,0.245638,6314.704590,83026.528405,3.407072,2018.767258,6.387904,4228.719666,0.003776,49.431458,108908.937500
+244,1.518198,1.683354,0.000206,0.267468,6348.657227,82582.502298,3.362176,2023.710220,6.377824,4257.041565,0.003264,50.179455,108908.937500
+245,1.521052,1.683354,0.000205,0.258090,6308.139648,83112.934909,3.337952,2015.468582,6.377344,4225.288757,0.003424,49.702145,108908.937500
+246,1.502181,1.683354,0.000204,0.197207,6346.196289,82614.526264,3.435584,2057.775986,6.372192,4221.345825,0.003328,49.333218,108908.937500
+247,1.523221,1.683354,0.000203,0.285573,6349.257812,82574.690693,3.260640,2065.137924,6.470144,4216.786865,0.003904,49.609280,108908.937500
+248,1.499708,1.683354,0.000202,0.295060,6318.238770,82980.086560,3.302720,2012.749054,6.381888,4238.344727,0.003424,49.505440,108908.937500
+249,1.498004,1.683354,0.000201,0.253159,6318.229004,82980.214816,3.328064,2011.248062,6.390752,4238.502075,0.003424,50.749313,108908.937500
+250,1.479036,1.505146,0.000200,0.261037,11387.784180,46039.509682,3.283712,2013.640686,6.374368,4229.850525,0.003520,49.612926,108908.937500
+251,1.524658,1.505146,0.000199,0.209427,6312.755371,83052.164892,3.284768,2007.267273,6.404320,4238.279602,0.003232,49.498016,108908.937500
+252,1.530877,1.505146,0.000198,0.316635,6324.155273,82902.455321,3.305088,2022.990784,6.376896,4234.181213,0.003392,49.334270,108908.937500
+253,1.498913,1.505146,0.000197,0.267035,6348.044434,82590.474198,3.275840,2048.158691,6.388704,4232.762024,0.003552,49.483746,108908.937500
+254,1.514113,1.505146,0.000196,0.257986,6325.698242,82882.233696,3.317184,2033.158966,6.385920,4225.396057,0.003872,49.443970,108908.937500
+255,1.534029,1.505146,0.000195,0.195580,6346.025391,82616.751073,3.295840,2024.884811,6.393824,4254.072632,0.002784,49.350079,108908.937500
+256,1.479002,1.505146,0.000194,0.214464,6320.090820,82955.769926,3.312064,2017.095886,6.372896,4236.192261,0.003456,49.170624,108908.937500
+257,1.474892,1.505146,0.000193,0.306380,6323.166016,82915.425390,3.301536,2013.714355,6.375456,4242.178284,0.003200,49.583679,108908.937500
+258,1.479650,1.505146,0.000192,0.271395,6328.001953,82852.060395,3.444896,2018.546371,6.391488,4242.194580,0.002976,49.466911,108908.937500
+259,1.469126,1.505146,0.000191,0.214684,6316.694336,83000.375215,3.358880,2010.288330,6.364768,4239.265259,0.003584,49.413090,108908.937500
+260,1.482932,1.505146,0.000190,0.370969,6349.332031,82573.725460,3.465216,2025.445480,6.387392,4256.749817,0.003744,49.321663,108908.937500
+261,1.472604,1.505146,0.000189,0.308654,6299.277832,83229.858085,3.369216,2014.045624,6.393248,4218.109131,0.003424,49.385887,108908.937500
+262,1.510692,1.505146,0.000187,0.220909,6306.563477,83133.706962,3.504064,2010.690781,6.377728,4228.630615,0.002976,49.424191,108908.937500
+263,1.521401,1.505146,0.000186,0.254238,6335.748047,82750.765359,3.466624,2042.329178,6.396448,4226.178528,0.003456,49.384991,108908.937500
+264,1.518334,1.505146,0.000185,0.267601,6346.579590,82609.536771,3.336832,2058.003632,6.383136,4221.394348,0.003360,49.419743,108908.937500
+265,1.467476,1.505146,0.000184,0.392532,6373.007812,82266.963328,3.345760,2068.048248,6.393120,4237.773743,0.003584,49.468288,108908.937500
+266,1.439376,1.505146,0.000183,0.203255,6320.787598,82946.625227,14.370080,2033.779266,6.378976,4208.961304,0.002976,49.307262,108908.937500
+267,1.543246,1.505146,0.000182,0.240690,6323.341797,82913.120442,3.285408,2019.077820,6.380480,4237.297363,0.003360,49.341953,108908.937500
+268,1.523543,1.505146,0.000181,0.202465,6298.321777,83242.491974,3.504992,2010.187637,6.362656,4221.047852,0.002880,49.308830,108908.937500
+269,1.496513,1.505146,0.000180,0.190458,6305.687500,83145.255771,3.415936,2014.631592,6.371840,4223.898987,0.003136,49.426334,108908.937500
+270,1.457562,1.505146,0.000179,0.213912,6299.397949,83228.271055,3.347488,2011.426453,6.373248,4220.979126,0.003264,49.328800,108908.937500
+271,1.535891,1.505146,0.000178,0.213410,6311.645996,83066.762668,3.375616,2015.071793,6.383808,4229.490723,0.002816,49.369663,108908.937500
+272,1.501040,1.505146,0.000177,0.218659,6319.931152,82957.865736,3.327264,2014.723389,6.388032,4238.305786,0.003360,49.251999,108908.937500
+273,1.566223,1.505146,0.000176,0.262533,6308.453613,83108.798469,3.303328,2019.352478,6.447200,4221.954956,0.003520,49.439362,108908.937500
+274,1.495419,1.505146,0.000175,0.226422,6317.879395,82984.806651,3.358496,2022.205902,6.371552,4228.497192,0.003808,49.492001,108908.937500
+275,1.529794,1.505146,0.000174,0.255577,6314.583496,83028.120592,3.409344,2021.723557,6.382624,4225.592712,0.003360,49.456383,108908.937500
+276,1.466739,1.505146,0.000173,0.194698,6323.101562,82916.270570,3.428512,2010.307938,6.397056,4245.558838,0.003392,49.465729,108908.937500
+277,1.534216,1.505146,0.000172,0.226489,6318.673828,82974.373146,3.272064,2007.580902,6.648800,4243.739136,0.003200,49.449249,108908.937500
+278,1.468664,1.505146,0.000171,0.213755,6335.289551,82756.754178,3.351904,2018.624069,6.383488,4249.254028,0.003488,49.694271,108908.937500
+279,1.551666,1.505146,0.000170,0.240478,6311.239746,83072.109616,3.356128,2018.430450,6.415904,4225.687439,0.003168,49.361473,108908.937500
+280,1.486062,1.505146,0.000169,0.312648,6307.027344,83127.592672,3.392032,2014.813858,6.369376,4225.182739,0.003296,49.301952,108908.937500
+281,1.491243,1.505146,0.000168,0.212957,6330.034180,82825.461146,3.441408,2013.178711,6.375328,4249.715576,0.003424,49.379360,108908.937500
+282,1.489311,1.505146,0.000167,0.201416,6292.832520,83315.104664,3.373344,2013.903229,6.370176,4211.797974,0.003488,49.447327,108908.937500
+283,1.416811,1.505146,0.000166,0.189320,6325.158203,82889.310143,3.446048,2031.153366,6.392928,4226.724976,0.003392,49.449760,108908.937500
+284,1.525040,1.505146,0.000165,0.191450,6333.959473,82774.132399,3.419392,2041.122589,6.401056,4225.420654,0.003264,49.548897,108908.937500
+285,1.497472,1.505146,0.000164,0.212068,6326.708984,82868.992599,3.383744,2041.351135,6.381504,4218.273682,0.003488,49.344032,108908.937500
+286,1.476106,1.505146,0.000163,0.212023,6310.102051,83087.087306,3.281664,2012.519913,6.369280,4230.531189,0.003680,49.443680,108908.937500
+287,1.430440,1.505146,0.000162,0.170482,6310.171875,83086.167918,3.333184,2015.513626,6.376480,4227.606995,0.003296,49.354527,108908.937500
+288,1.457449,1.505146,0.000161,0.210575,6313.653320,83040.352931,3.401632,2015.328201,6.390336,4231.105835,0.003200,49.479614,108908.937500
+289,1.432535,1.505146,0.000160,0.160096,6317.718750,82986.916757,3.296000,2011.740524,6.368224,4238.648621,0.003296,49.698719,108908.937500
+290,1.415617,1.505146,0.000159,0.345268,6311.401367,83069.982322,3.495328,2016.944962,6.371488,4227.352966,0.002976,49.308865,108908.937500
+291,1.482946,1.505146,0.000158,0.183449,6309.170410,83099.356320,3.319424,2009.765244,6.372480,4232.322876,0.003648,49.406113,108908.937500
+292,1.491100,1.505146,0.000157,0.231720,6353.942871,82513.804521,3.363360,2082.302475,6.371360,4204.531982,0.003200,49.451393,108908.937500
+293,1.501492,1.505146,0.000156,0.179347,6313.354492,83044.283455,3.351936,2013.163208,6.388224,4232.997986,0.003040,49.426239,108908.937500
+294,1.435696,1.505146,0.000156,0.185050,6319.116699,82968.557942,3.355872,2018.034958,6.362848,4233.572510,0.003040,49.765953,108908.937500
+295,1.491356,1.505146,0.000155,0.182713,6321.441895,82938.039888,3.385440,2015.165497,6.372096,4239.233337,0.003584,49.237888,108908.937500
+296,1.504897,1.505146,0.000154,0.233660,6323.807129,82907.019350,3.345248,2018.926804,6.369600,4236.470520,0.002624,50.708286,108908.937500
+297,1.437167,1.505146,0.000153,0.164880,6315.659180,83013.979235,3.521984,2015.129181,6.372224,4230.859192,0.003552,51.784321,108908.937500
+298,1.450542,1.505146,0.000152,0.264929,6318.756348,82973.289545,3.410592,2012.532181,6.380224,4237.041321,0.003264,51.380161,108908.937500
+299,1.449647,1.505146,0.000151,0.178139,6340.149414,82693.319315,3.426272,2048.857025,6.371520,4222.619141,0.003808,50.868927,108908.937500
+300,1.499225,1.505146,0.000150,0.253062,6180.783691,84825.489157,3.284192,1987.506058,6.376800,4126.161163,0.003456,49.417599,108908.937500
+301,1.472106,1.505146,0.000149,0.164925,6223.688965,84240.713661,3.260192,1990.077438,6.369760,4166.303894,0.002976,49.760098,108908.937500
+302,1.473293,1.505146,0.000148,0.197588,6230.845215,84143.961521,3.224576,1996.359634,6.357248,4167.552063,0.003648,49.405025,108908.937500
+303,1.397302,1.505146,0.000147,0.203716,6252.905762,83847.097650,3.193440,2001.285583,6.384192,4184.667603,0.003168,49.422943,108908.937500
+304,1.491825,1.505146,0.000146,0.186111,6283.622070,83437.226831,3.338368,2008.019119,6.397440,4208.341919,0.003328,49.617634,108908.937500
+305,1.508419,1.505146,0.000146,0.183281,6303.931152,83168.420995,3.306784,2002.922745,6.384480,4233.988342,0.003456,49.456383,108908.937500
+306,1.434751,1.505146,0.000145,0.166023,6309.978027,83088.720393,3.333248,2012.737411,6.384320,4230.249573,0.003136,49.355072,108908.937500
+307,1.530075,1.505146,0.000144,0.201598,6320.490234,82950.527658,3.286976,2037.537384,6.399712,4215.846924,0.003008,49.431774,108908.937500
+308,1.515311,1.505146,0.000143,0.227476,6381.569336,82156.593841,3.445632,2069.454895,6.410272,4244.910950,0.003040,49.406273,108908.937500
+309,1.459164,1.505146,0.000142,0.180775,6342.942383,82656.907214,3.278240,2013.942810,6.393312,4261.398132,0.003616,50.071903,108908.937500
+310,1.412815,1.505146,0.000141,0.184167,6327.155273,82863.147393,3.329632,2014.858856,6.394368,4245.103088,0.003488,49.545822,108908.937500
+311,1.483627,1.505146,0.000140,0.215028,6374.019043,82253.911773,3.812064,2022.608398,6.411392,4283.865601,0.003168,49.380352,108908.937500
+312,1.445003,1.505146,0.000140,0.188585,6322.283203,82927.003292,3.261024,2016.482956,6.459296,4238.798340,0.003520,49.336479,108908.937500
+313,1.427851,1.505146,0.000139,0.226719,6410.135254,81790.473872,3.349152,2069.109833,6.387424,4273.969971,0.003360,49.447071,108908.937500
+314,1.478248,1.505146,0.000138,0.323890,6359.861328,82437.017562,3.237568,2053.672546,6.416736,4239.256409,0.003392,49.348866,108908.937500
+315,1.486552,1.505146,0.000137,0.146325,6339.509766,82701.662965,3.362432,2015.786438,6.399872,4256.613647,0.002944,49.472446,108908.937500
+316,1.465203,1.505146,0.000136,0.189167,6354.064941,82512.219317,3.299520,2025.431412,6.696256,4261.310608,0.003584,49.363041,108908.937500
+317,1.453553,1.505146,0.000135,0.199033,6386.510742,82093.027189,3.307808,2085.489273,6.393760,4233.905212,0.003456,49.553024,108908.937500
+318,1.425427,1.505146,0.000135,0.190021,6341.979492,82669.456854,3.230080,2030.546082,6.388640,4244.534912,0.003200,49.349312,108908.937500
+319,1.531694,1.505146,0.000134,0.174893,6324.084961,82903.377048,3.429536,2018.689880,6.412640,4238.261780,0.003648,49.347935,108908.937500
+320,1.471653,1.505146,0.000133,0.185825,6325.791992,82881.005358,3.318464,2026.437210,6.401664,4231.702820,0.002880,49.402466,108908.937500
+321,1.536368,1.505146,0.000132,0.172649,6321.286621,82940.077144,3.256224,2009.725327,6.37

logs/in2_train_csv/CP_Training_log13_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 100
+  run: 13
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log14.csv

@@ -0,0 +1,118 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,1.920389,1.888691,0.000003,1.139446,16001.439453,32765.052265,2.924960,4222.877640,6.421504,4663.327698,0.003712,62.615009,107212.937500
+1,1.898834,1.888691,0.000006,1.156716,8280.115234,63318.925541,4.027040,3667.506577,6.380544,4544.214996,0.003840,49.815872,108908.937500
+2,1.910765,1.888691,0.000009,1.280252,9915.143555,52877.499666,2.557024,5223.525711,6.353568,4624.979370,0.003520,49.617279,108908.937500
+3,1.936233,1.888691,0.000012,1.266597,6216.728027,84335.038897,2.602912,1991.833389,6.372544,4158.053223,0.003744,49.909855,108908.937500
+4,1.905092,1.888691,0.000015,1.160154,6238.630371,84038.958684,3.123872,1995.341507,6.390656,4176.538696,0.003392,49.345730,108908.937500
+5,1.966748,1.888691,0.000018,1.149405,6276.423828,83532.918483,3.054464,2005.503036,6.367168,4204.320740,0.003712,49.326176,108908.937500
+6,1.904285,1.888691,0.000021,1.043706,6345.527344,82623.235485,3.235392,2013.193588,6.432832,4265.340576,0.003584,49.419582,108908.937500
+7,1.854531,1.888691,0.000024,1.021607,6402.649902,81886.095288,3.106528,2033.093323,6.384544,4302.618103,0.003872,49.572575,108908.937500
+8,1.830776,1.888691,0.000027,0.987560,6419.801270,81667.325512,3.049472,2037.534195,6.417824,4315.349182,0.003776,49.563393,108908.937500
+9,1.852669,1.888691,0.000030,0.884296,6426.617676,81580.704882,3.127136,2048.292542,6.397856,4311.555664,0.003648,49.356930,108908.937500
+10,1.806110,1.888691,0.000033,0.829637,6450.763184,81275.344495,3.055648,2058.223145,6.712832,4325.536438,0.003744,49.371647,108908.937500
+11,1.792623,1.888691,0.000036,1.050088,6435.643066,81466.295534,3.116672,2058.158371,6.560000,4310.119324,0.003424,49.848480,108908.937500
+12,1.784030,1.888691,0.000039,1.042397,6422.322754,81635.261897,3.118080,2025.747787,6.376992,4329.702515,0.003328,49.523106,108908.937500
+13,1.786141,1.888691,0.000042,1.344665,6375.977539,82228.646006,3.168352,2025.594009,6.385952,4283.134155,0.003232,49.667774,108908.937500
+14,1.809938,1.888691,0.000045,1.294165,6358.028320,82460.784002,3.099840,2027.823807,6.402336,4263.457825,0.003584,49.421951,108908.937500
+15,1.793668,1.888691,0.000048,1.255733,6368.768066,82321.729184,3.235840,2042.357132,6.383040,4259.310303,0.003648,49.683136,108908.937500
+16,1.765961,1.888691,0.000051,1.088759,6343.388184,82651.098250,3.196384,2021.261993,6.380608,4254.785339,0.003328,49.924606,108908.937500
+17,1.712484,1.888691,0.000054,0.790309,6323.919434,82905.547028,3.168512,2022.918533,6.392640,4234.136108,0.003392,49.469185,108908.937500
+18,1.731035,1.888691,0.000057,0.858386,6329.273438,82835.416289,3.023552,2017.821060,6.566048,4244.642883,0.003520,49.324898,108908.937500
+19,1.743202,1.888691,0.000060,0.738515,6334.254883,82770.272068,3.186336,2031.902344,6.383616,4235.263062,0.003808,49.672001,108908.937500
+20,1.718590,1.888691,0.000063,1.169419,6330.504883,82819.302679,3.075872,2022.275513,6.377664,4241.392334,0.003168,49.555038,108908.937500
+21,1.767774,1.888691,0.000066,0.992476,6338.120605,82719.789136,3.131392,2018.031113,6.384160,4253.191162,0.003776,49.561214,108908.937500
+22,1.697396,1.888691,0.000069,0.690903,6326.690430,82869.235634,3.174304,2019.088104,6.386336,4240.617859,0.003232,49.572674,108908.937500
+23,1.707077,1.888691,0.000072,0.763766,6371.410645,82287.585788,3.158560,2050.045227,6.402144,4254.645264,0.003424,49.334175,108908.937500
+24,1.671977,1.888691,0.000075,0.777583,6375.658691,82232.758273,3.250304,2069.236313,6.380576,4239.289062,0.003264,49.613728,108908.937500
+25,1.739844,1.888691,0.000078,0.823003,6332.830566,82788.887923,3.150048,2010.980927,6.399552,4254.874023,0.003232,49.569569,108908.937500
+26,1.652054,1.888691,0.000081,0.907736,6340.053223,82694.573939,3.147936,2025.912552,6.386656,4247.142090,0.003424,49.651134,108908.937500
+27,1.654510,1.888691,0.000084,0.848301,6341.171875,82679.985708,3.139616,2023.191956,6.380128,4250.949646,0.003776,49.672672,108908.937500
+28,1.691144,1.888691,0.000087,0.762472,6357.385742,82469.118795,3.161920,2045.897415,6.377600,4244.729370,0.003616,49.393024,108908.937500
+29,1.663364,1.888691,0.000090,0.902108,6359.409668,82442.872432,3.139968,2024.028030,6.401888,4268.531555,0.003296,49.442944,108908.937500
+30,1.633048,1.888691,0.000093,0.708434,6332.022949,82799.447223,3.128320,2011.973206,6.372000,4253.224731,0.003680,49.483425,108908.937500
+31,1.700897,1.888691,0.000096,0.594090,6339.412598,82702.930583,3.130432,2024.291840,6.383232,4248.282654,0.003840,49.497410,108908.937500
+32,1.608716,1.888691,0.000099,0.632816,6395.619629,81976.107152,3.177792,2066.547470,6.404288,4262.028076,0.003424,49.590847,108908.937500
+33,1.657396,1.888691,0.000102,0.638838,6355.234863,82497.029815,3.097952,2024.498062,6.382944,4263.747192,0.003680,49.657726,108908.937500
+34,1.648264,1.888691,0.000105,0.633965,6350.555176,82557.821401,3.142144,2026.387573,6.398528,4257.194885,0.003584,49.585888,108908.937500
+35,1.660285,1.888691,0.000108,0.775887,6348.233398,82588.015767,3.069088,2080.110306,6.365472,4201.257446,0.003616,49.536991,108908.937500
+36,1.631099,1.888691,0.000111,0.584269,6381.269531,82160.453720,3.076288,2063.766998,6.383360,4250.165283,0.003232,49.998657,108908.937500
+37,1.626856,1.888691,0.000114,0.680088,6330.014160,82825.723092,3.160064,2013.283936,6.393504,4249.904907,0.003488,49.434078,108908.937500
+38,1.619444,1.888691,0.000117,0.652532,6337.687500,82725.442048,3.152288,2020.087616,6.378720,4250.564758,0.003392,49.675808,108908.937500
+39,1.642475,1.888691,0.000120,0.772394,6377.823242,82204.849537,3.183968,2075.630951,6.376000,4235.463989,0.003872,49.339359,108908.937500
+40,1.600698,1.888691,0.000123,0.854414,6376.743652,82218.766910,3.233664,2071.577423,6.401248,4237.967407,0.003424,49.717503,108908.937500
+41,1.580887,1.888691,0.000126,0.981567,6324.282715,82900.784743,3.075968,2022.029312,6.382144,4235.373962,0.003584,49.572098,108908.937500
+42,1.624956,1.888691,0.000129,0.872412,6332.328125,82795.456845,3.078176,2018.147095,6.668672,4246.871277,0.003040,49.719967,108908.937500
+43,1.623077,1.888691,0.000132,0.637372,6346.729980,82607.579275,3.262944,2024.297745,6.368064,4255.354919,0.003360,49.534527,108908.937500
+44,1.554411,1.888691,0.000135,0.831163,6327.952637,82852.706096,3.273952,2012.035049,6.378240,4248.672424,0.003520,49.655296,108908.937500
+45,1.633371,1.888691,0.000138,0.773916,6346.301758,82613.153299,3.143520,2037.514160,6.385248,4241.558044,0.003392,49.806145,108908.937500
+46,1.607605,1.888691,0.000141,0.742652,6313.643066,83040.487795,3.063552,2018.990524,6.388064,4227.843445,0.003456,49.511295,108908.937500
+47,1.635939,1.888691,0.000144,0.691577,6347.576172,82596.566911,3.112992,2024.110214,6.399936,4256.378906,0.003808,49.718784,108908.937500
+48,1.633764,1.888691,0.000147,0.804403,6336.690918,82738.452417,3.320224,2021.176804,6.378688,4248.116577,0.003328,49.573761,108908.937500
+49,1.607873,1.888691,0.000150,0.621125,6305.447266,83148.423564,3.180544,2018.955246,6.371584,4219.534424,0.003648,49.582783,108908.937500
+50,1.639636,1.888691,0.000153,0.711546,6370.114746,82304.325887,3.150752,2078.611542,6.378656,4224.542603,0.003168,49.543327,108908.937500
+51,1.656035,1.888691,0.000156,0.796302,6345.472168,82623.953919,3.182784,2031.570267,6.604704,4246.810364,0.003456,49.479839,108908.937500
+52,1.662412,1.888691,0.000159,0.705655,6316.856445,82998.245178,3.135296,2016.884125,6.378144,4232.103760,0.003520,49.502239,108908.937500
+53,1.581693,1.888691,0.000162,0.666786,6342.799805,82658.765237,3.154784,2028.590302,6.382144,4247.130981,0.003456,49.700607,108908.937500
+54,1.594439,1.888691,0.000165,0.847070,6319.421875,82964.551247,3.180128,2024.629105,6.375776,4227.748169,0.003680,49.578304,108908.937500
+55,1.670181,1.888691,0.000168,0.809112,6371.773926,82282.894231,3.222016,2057.925903,6.390016,4246.990723,0.003520,49.358879,108908.937500
+56,1.632861,1.888691,0.000171,0.773693,6320.738281,82947.272403,3.153024,2018.593735,6.379328,4235.251831,0.003232,49.522881,108908.937500
+57,1.572145,1.888691,0.000174,0.892174,6340.897461,82683.563838,3.247520,2020.747467,6.376864,4253.231934,0.003328,49.488415,108908.937500
+58,1.589554,1.888691,0.000177,0.677442,6324.347656,82899.933479,3.095200,2026.461670,6.375456,4230.713440,0.003424,49.848511,108908.937500
+59,1.586414,1.888691,0.000180,0.757773,6376.862305,82217.237091,3.191552,2054.159515,6.498272,4255.847351,0.003264,49.321217,108908.937500
+60,1.630998,1.888691,0.000183,0.753521,6323.771484,82907.486663,3.224288,2018.225296,6.375328,4238.561157,0.002944,49.535168,108908.937500
+61,1.556511,1.888691,0.000186,0.894534,6345.840820,82619.154001,3.159488,2025.426559,6.395200,4251.989136,0.003328,50.869598,108908.937500
+62,1.594290,1.888691,0.000189,0.849950,6348.363281,82586.326077,3.166304,2026.310699,6.374784,4255.094788,0.003360,49.573601,108908.937500
+63,1.591010,1.888691,0.000192,0.721222,6382.641113,82142.798051,3.106112,2080.148941,6.377664,4235.520752,0.003712,49.620064,108908.937500
+64,1.558378,1.888691,0.000195,0.834214,6370.168457,82303.631927,3.276832,2050.656693,6.605248,4252.439270,0.003232,49.351906,108908.937500
+65,1.605214,1.888691,0.000198,0.679925,6333.108887,82785.249611,3.083712,2021.721039,6.377664,4244.510315,0.003264,49.538914,108908.937500
+66,1.590822,1.888691,0.000201,0.932467,6346.551758,82609.899046,3.153696,2013.050842,6.388480,4266.570618,0.003456,49.556160,108908.937500
+67,1.622504,1.888691,0.000204,0.886735,6339.600098,82700.484561,3.192448,2021.613373,6.369120,4250.808960,0.003360,49.761311,108908.937500
+68,1.598213,1.888691,0.000207,0.945386,6362.027832,82408.944733,3.032032,2052.177292,6.408832,4242.855530,0.003136,49.706528,108908.937500
+69,1.635720,1.888691,0.000210,0.828467,6375.836914,82230.459635,3.054080,2071.758331,6.374304,4237.343994,0.003488,49.471584,108908.937500
+70,1.608484,1.888691,0.000213,0.705563,6333.446777,82780.833002,3.085312,2030.654495,6.383808,4235.660950,0.003456,49.810143,108908.937500
+71,1.609575,1.888691,0.000216,0.769313,6333.448730,82780.807473,3.185920,2022.177628,6.377888,4244.523987,0.003744,49.391457,108908.937500
+72,1.544902,1.888691,0.000219,0.765577,6330.444336,82820.094796,2.990560,2021.862930,6.384096,4241.289978,0.003456,50.048832,108908.937500
+73,1.616233,1.888691,0.000222,0.908922,6347.528809,82597.183220,3.052000,2046.735794,6.404224,4233.970032,0.003360,49.475582,108908.937500
+74,1.594353,1.888691,0.000225,0.870625,6332.663574,82791.071064,3.016256,2023.631378,6.390528,4242.107483,0.003648,49.662495,108908.937500
+75,1.609479,1.888691,0.000228,0.901361,6339.545898,82701.191599,3.144448,2023.043823,6.556544,4249.322388,0.003168,49.655582,108908.937500
+76,1.552467,1.888691,0.000231,0.720024,6335.944336,82748.201720,3.141376,2018.586441,6.378432,4250.480408,0.003232,49.533215,108908.937500
+77,1.623230,1.888691,0.000234,0.868704,6327.529297,82858.249311,3.159808,2033.299561,6.390720,4227.265991,0.003296,49.573982,108908.937500
+78,1.589706,1.888691,0.000237,0.753138,6334.817383,82762.922483,3.093888,2022.446396,6.369728,4245.048401,0.003168,50.036736,108908.937500
+79,1.593231,1.888691,0.000240,0.930943,6320.514160,82950.213656,3.199552,2013.510864,6.400224,4239.887573,0.003616,49.629375,108908.937500
+80,1.596107,1.888691,0.000243,1.606060,6329.073730,82838.030070,3.101120,2014.925446,6.389056,4247.571899,0.002752,49.241570,108908.937500
+81,1.639003,1.888691,0.000246,0.817727,6320.255859,82953.603725,3.115648,2019.156494,6.376960,4234.370728,0.003744,49.385345,108908.937500
+82,1.667713,1.888691,0.000249,0.836137,6397.347168,81953.970331,3.113664,2098.460571,6.379456,4232.184082,0.003520,49.324863,108908.937500
+83,1.597706,1.888691,0.000252,0.850811,6376.568848,82221.020823,3.066400,2045.288528,6.396896,4264.504700,0.003680,49.474209,108908.937500
+84,1.606992,1.888691,0.000255,0.882354,6307.102539,83126.601598,3.170624,2014.658478,6.425856,4225.344910,0.003040,49.490398,108908.937500
+85,1.606081,1.888691,0.000258,0.866452,6330.486816,82819.539035,3.189280,2021.691483,6.376000,4241.203735,0.003712,50.007488,108908.937500
+86,1.608906,1.888691,0.000261,0.911513,6355.762207,82490.184957,3.082752,2027.016525,6.386272,4261.731934,0.003744,49.692703,108908.937500
+87,1.671125,1.888691,0.000264,0.814413,6334.824219,82762.833174,3.132576,2028.304459,6.405984,4239.682617,0.003552,49.423584,108908.937500
+88,1.673897,1.888691,0.000267,0.830793,6355.620117,82492.029154,3.088928,2057.730515,6.397024,4230.854980,0.003488,49.693249,108908.937500
+89,1.632511,1.888691,0.000270,0.727420,6347.372559,82599.216473,3.271584,2019.465515,6.377504,4260.599365,0.003456,49.754658,108908.937500
+90,1.688837,1.888691,0.000273,0.829598,6319.859863,82958.801515,3.233824,2016.384445,6.386720,4236.367371,0.003712,49.533119,108908.937500
+91,1.636074,1.888691,0.000276,0.817738,6328.082031,82851.011951,3.357184,2024.802856,6.504896,4236.096619,0.003936,49.416801,108908.937500
+92,1.643463,1.888691,0.000279,0.835200,6385.577637,82105.023199,3.101984,2085.348373,6.370688,4233.383240,0.003264,49.486050,108908.937500
+93,1.674679,1.888691,0.000282,0.844461,6345.835449,82619.223930,3.206112,2050.609604,6.416192,4228.289124,0.003296,49.477825,108908.937500
+94,1.552707,1.888691,0.000285,0.863358,6333.635742,82778.363225,3.077472,2024.566391,6.388608,4242.301086,0.003520,49.442337,108908.937500
+95,1.630905,1.888691,0.000288,0.832031,6353.830566,82515.262962,3.274624,2028.155670,6.383200,4258.484619,0.003584,49.571232,108908.937500
+96,1.621827,1.888691,0.000291,0.918384,6323.018066,82917.365488,3.162592,2012.941772,6.531904,4242.120300,0.003328,50.340000,108908.937500
+97,1.617399,1.888691,0.000294,0.769910,6362.215332,82406.516070,3.146368,2055.820648,6.363808,4239.379150,0.003360,49.700993,108908.937500
+98,1.612508,1.888691,0.000297,0.735902,6338.875488,82709.938217,3.280320,2024.479431,6.381600,4246.271057,0.003648,49.950272,108908.937500
+99,1.637773,1.888691,0.000300,0.846796,6349.173828,82575.782959,3.284096,2017.665619,6.522688,4264.178955,0.003808,49.582047,108908.937500
+100,1.653108,1.888691,0.000300,0.734803,6202.841309,84523.845431,3.153216,1994.373230,6.355648,4141.351471,0.003040,49.651264,108908.937500
+101,1.685746,1.888691,0.000300,0.839908,6250.926758,83873.643111,3.112736,2000.709305,6.356064,4183.176941,0.003328,49.745953,108908.937500
+102,1.593222,1.888691,0.000300,0.901897,6296.575684,83265.575822,3.109248,2014.010040,6.365856,4215.727234,0.003648,49.486912,108908.937500
+103,1.654905,1.888691,0.000300,1.344759,6319.219238,82967.211649,3.203072,2014.496460,6.400864,4237.814575,0.003904,49.331680,108908.937500
+104,1.717948,1.888691,0.000300,0.872429,6307.382812,83122.907803,3.308448,2013.957855,6.368896,4226.038208,0.003424,49.683266,108908.937500
+105,1.634465,1.888691,0.000300,0.930153,6328.196289,82849.516047,3.181632,2020.862473,6.370432,4239.761902,0.003584,49.982849,108908.937500
+106,1.635792,1.888691,0.000300,0.841532,6351.314453,82547.951903,3.118720,2022.737396,6.384096,4261.141296,0.003520,50.008610,108908.937500
+107,1.702721,1.888691,0.000300,1.061945,6335.490723,82754.126389,3.155392,2014.187668,6.598240,4254.034668,0.003680,49.600864,108908.937500
+108,1.590036,1.888691,0.000300,0.763811,6324.348145,82899.927078,3.180352,2017.641800,6.588832,4239.494141,0.003232,49.576927,108908.937500
+109,1.636682,1.888691,0.000300,0.749749,6365.064453,82369.629383,3.136064,2047.121140,6.403680,4250.903137,0.003648,49.604511,108908.937500
+110,1.652374,1.888691,0.000299,0.730469,6359.887695,82436.675790,3.187968,2034.555664,6.399360,4258.320068,0.003584,49.594624,108908.937500
+111,1.643456,1.888691,0.000299,0.650920,6332.087891,82798.598038,3.185760,2019.466751,6.392064,4245.492798,0.003648,49.602943,108908.937500
+112,1.615295,1.888691,0.000299,0.671301,6326.246582,82875.049716,3.325024,2020.936401,6.384960,4238.044067,0.003072,49.568512,108908.937500
+113,1.605785,1.888691,0.000299,0.768448,6312.109863,83060.658220,3.108032,2010.817078,6.372480,4234.687378,0.003488,49.210400,108908.937500
+114,1.643168,1.888691,0.000299,0.754148,6342.302246,82665.249882,3.096288,2018.112579,6.384480,4257.198425,0.003360,49.664864,108908.937500
+115,1.622634,1.888691,0.000299,0.693109,6359.413574,82442.821792,3.086368,2020.846024,6.413856,4270.722473,0.003456,50.430367,108908.937500
+116,1.572031,1.888691,0.000299,0.787975,6342.383301,82664.193433,3.351072,2027.193253,6.384608,4247.884216,0.003648,49.60

logs/in2_train_csv/CP_Training_log14_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 100
+  run: 14
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log1_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 4096
+  vocab_size: 50304  (tying=0)
+  B: 1
+  T: 4096
+  world_size: 1  (dp=1 x cp=1)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 32768
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 12
+  warmup_steps: 91
+  checkpointing: 0
+  run: 1
+  GPU Memory Used (rank 0): 38698.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log2.csv

@@ -0,0 +1,5 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.749795,-1.000000,0.000006,0.369926,19070.187500,54985.091258,28.417728,9397.607430,23.844800,9498.981613,0.003232,61.891327,52292.937500
+1,2.768989,-1.000000,0.000012,0.369283,15180.346680,69074.575313,22.225088,5218.900940,15.223200,9833.401306,0.003392,48.674625,52708.937500
+2,2.739640,-1.000000,0.000018,0.367964,15601.478516,67210.040314,22.717760,5712.806496,15.106144,9759.792923,0.003072,48.829056,52644.937500
+3,2.713684,-1.000000,0.000024,0.336673,13912.223633,75370.841332,21.858464,4363.964279,15.188960,9419.757721,0.003360,48.711327,52644.937500

logs/in2_train_csv/CP_Training_log2_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 1
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 64
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 4
+  warmup_steps: 1
+  checkpointing: 0
+  run: 2
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log3.csv

@@ -0,0 +1,5 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.749795,-1.000000,0.000006,0.369922,19128.255859,54818.170967,18.908096,9988.749802,21.508704,8997.138885,0.003328,62.464226,68220.937500
+1,2.769008,-1.000000,0.000012,0.369318,15397.692383,68099.555046,12.860640,6346.339447,13.852000,8952.085632,0.003232,49.240578,69148.937500
+2,2.739645,-1.000000,0.000018,0.368019,15472.694336,67769.450959,13.329216,6811.885391,13.934272,8560.524719,0.003264,49.327934,69180.937500
+3,2.713675,-1.000000,0.000024,0.336673,12844.812500,81634.200577,12.084192,4144.275375,13.874944,8601.305389,0.003648,49.334496,69180.937500

logs/in2_train_csv/CP_Training_log3_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 2
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 32
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 4
+  warmup_steps: 1
+  checkpointing: 0
+  run: 3
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log4.csv

@@ -0,0 +1,5 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.749795,-1.000000,0.000006,0.369903,23021.306641,45548.066249,13.072288,13853.379288,21.374208,9039.333008,0.003712,62.146366,108268.937500
+1,2.769012,-1.000000,0.000012,0.369288,18315.851562,57249.645010,8.078784,9006.841492,13.126144,9223.936829,0.003200,48.987518,109836.937500
+2,2.739646,-1.000000,0.000018,0.368004,17922.433594,58506.340365,8.335008,9489.957993,12.847552,8347.856201,0.003552,49.063423,109836.937500
+3,2.713669,-1.000000,0.000024,0.336734,12403.550781,84538.372801,7.035776,3995.475113,12.814368,8324.248291,0.003936,49.441536,109836.937500

logs/in2_train_csv/CP_Training_log4_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 4
+  warmup_steps: 1
+  checkpointing: 0
+  run: 4
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log5.csv

@@ -0,0 +1,9 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.749795,-1.000000,0.000006,0.369919,22952.294922,45685.017710,14.615648,13298.059341,21.100160,9524.594025,0.003488,62.183903,108268.937500
+1,2.768992,-1.000000,0.000012,0.369265,18392.845703,57009.992740,8.367776,9479.576767,12.932864,8828.016602,0.003648,49.494240,109836.937500
+2,2.739651,-1.000000,0.000018,0.368005,17966.494141,58362.860990,9.181056,9251.385132,12.969472,8629.169861,0.003232,49.257664,109836.937500
+3,2.713682,-1.000000,0.000024,0.336715,12413.190430,84472.723265,7.514432,4000.734772,12.974656,8328.348450,0.004000,49.090881,109836.937500
+4,2.788266,-1.000000,0.000030,0.295794,12557.221680,83503.821685,7.738080,4036.050385,13.020768,8436.570801,0.003872,49.207104,109836.937500
+5,2.807204,-1.000000,0.000036,0.244151,12679.691406,82697.280746,7.583680,4059.303848,13.029600,8536.062012,0.003616,49.192959,109836.937500
+6,2.819777,-1.000000,0.000042,0.235027,12614.707031,83123.293898,7.491168,4046.555313,13.350016,8483.425964,0.003328,49.362305,109836.937500
+7,2.751221,-1.000000,0.000048,0.202624,12584.532227,83322.604378,7.517024,4049.294739,13.011776,8450.923096,0.003680,49.228416,109836.937500

logs/in2_train_csv/CP_Training_log5_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 8
+  warmup_steps: 1
+  checkpointing: 0
+  run: 5
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log6_config.txt

@@ -0,0 +1,23 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ulysses  (load_balance=contiguous)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 8
+  warmup_steps: 1
+  checkpointing: 0
+  run: 6
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log7_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 750
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 250
+  run: 7
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log8_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 4096
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 4096
+  world_size: 4  (dp=4 x cp=1)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 1048576
+  grad_accum_steps: 16
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 1000
+  warmup_steps: 100
+  checkpointing: 1
+  ckpt_freq: 25
+  run: 8
+  GPU Memory Used (rank 0): 20906.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log9.csv

@@ -0,0 +1,3 @@
+step,train_loss,val_loss,lr,grad_norm,dt_ms,tok_per_sec,timer_data,timer_fwd,timer_loss,timer_bwd,timer_clip,timer_optim,mem_gpu_mb
+0,2.698207,-1.000000,0.000600,4.494938,1254.051147,6532.428934,7.251200,960.745605,3.987392,231.429344,15.948480,14.672000,36700.875000
+1,7.925948,-1.000000,0.000600,19.026079,342.769135,23899.468111,0.280864,102.697758,0.481152,209.203171,15.925248,11.102272,50076.875000

logs/in2_train_csv/CP_Training_log9_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 4096
+  vocab_size: 50304  (tying=0)
+  B: 1
+  T: 4096
+  world_size: 1  (dp=1 x cp=1)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 8192
+  grad_accum_steps: 2
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.0006
+  Min Learning Rate: 6e-05
+  max_steps: 2
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 50
+  run: 9
+  GPU Memory Used (rank 0): 9096.9 MB / 143155.9 MB