Upload folder using huggingface_hub

logs/in2_train_csv/CP_Training_log15_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 200
+  run: 15
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB

logs/in2_train_csv/CP_Training_log16_config.txt

@@ -0,0 +1,24 @@
+Configuration (bluscriptCP):
+  d_model: 1536
+  n_layers: 36
+  q_heads: 12
+  kv_heads: 4  (G=3, head_dim=128)
+  ffn_hidden: 4096
+  context_length: 16384
+  vocab_size: 50304  (tying=0)
+  B: 4
+  T: 16384
+  world_size: 4  (dp=1 x cp=4)
+  CP_ATTN_MODE: ring  (load_balance=HeadTail)
+  global_batch: 524288
+  grad_accum_steps: 8
+  Parameters: 1060624896
+  Parameters per GPU: 1060624896
+  Max Learning Rate: 0.001
+  Min Learning Rate: 0.0001
+  max_steps: 400
+  warmup_steps: 1
+  checkpointing: 1
+  ckpt_freq: 200
+  run: 16
+  GPU Memory Used (rank 0): 20904.9 MB / 143155.9 MB