Upload folder using huggingface_hub

logs/probe_run8/csv/1b_run8_T4096_ntk_absent_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_absent_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_absent_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_absent_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_absent_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_absent_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_absent_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_absent_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_absent_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_absent_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_absent_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_absent_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_null_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_null_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_null_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_null_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_null_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_null_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_null_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_null_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_null_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_null_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_null_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_null_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.25_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.25_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.25_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.25_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.25_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.25_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.25_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.25_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.25_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.25_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.25_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.25_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.5_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.5_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.5_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.5_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.5_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.5_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.5_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.5_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.5_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.5_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.5_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.5_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.75_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.75_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.75_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.75_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.75_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.75_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.75_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.75_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0.75_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0.75_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0.75_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0.75_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p0_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p0_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p0_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p0_b2.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p1_b0.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p1_b0.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p1_b0.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p1_b0.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p1_b1.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p1_b1.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p1_b1.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p1_b1.csv (4096 positions, 8 windows)

logs/probe_run8/csv/1b_run8_T4096_ntk_p1_b2.log

@@ -0,0 +1,45 @@
+=== Llama Context Parallel Training Script (bluscriptCP) ===
+[CP attn fusion] split (Stage-2 rope-once + Stage-3 Hopper causal-attn kernels)
+[LongRoPE] factors=probe_run8/needles/lam_ntk_hd128_th500000_s1.txt n_hat=0 s=1 S_search=4096 (alone)
+================ Configuration (bluscriptCP) ================
+  d_model/layers  : 1536 / 36
+  q/kv heads      : 12 / 4  (G=3, hd=128)
+  ffn_hidden      : 4096 (SwiGLU)
+  context_length  : 4096
+  vocab_size      : 50304  tying=0
+  B x T           : 1 x 4096
+  world_size      : 1  (dp=1 x cp=1)
+  CP_ATTN_MODE    : ring  (load_balance=HeadTail)
+  global_batch    : 524288  (grad_accum=128)
+  optimizer       : AdamW  [DDP]
+  precision       : fp32 everywhere
+  max_lr/min_lr   : 0.0006 / 6e-05
+  warmup/steps    : 91 / 917  (cosine, no stable phase)
+  checkpointing   : true  (ckpt_freq=50, dir=checkpoints_bluscriptcp)
+=============================================================
+Initializing model on CUDA device 0...
+[YaRN] build_rope_cache: seq_len=4096 hd=128 scale=1.000 low=7 high=25 m=1.0000
+[LongRoPE] build_rope_cache_longrope: seq_len=4096 hd=128 n_hat=0 lam0=1.0000 lam_last=1.0000 mscale=1.0000
+[InitFrom] weights loaded from /root/Adhi/BluTrain/dist/Context_Parallelism/checkpoints_bluscriptcp/blumodelcp_run8_step_1000.ckpt  (base step 1000, loss 2.66226)  -- optimizer state and RNG intentionally NOT restored; pair with CP_REWARMUP
+Parameters: 1060624896
+Parameters per GPU: 1060624896
+max_steps: 917
+warmup_steps: 91
+found 100 shards for split train
+found 1 shards for split val
+[Resume] latest run 8 already complete at step 1003; starting a new run.
+Saving logs to: CP_BluScreipt_Training_logs/CP_Training_log7.csv (run 7)
+[rope_theta] 500000 compiled default (no sidecar -- legacy run, assuming it was trained at this base)
+Checkpointing: ON -> checkpoints_bluscriptcp/blumodelcp_run7_step_<N>.ckpt (every 50 steps, keep 6)
+[eval] shared token stream: 32769 int32 tokens from probe_run8/needles/needle_T4096_p1_b2.bin (B forced to 1 for HF-scorer parity)
+[eval] PPL-vs-position: 8 windows (skip=0) x B=1 Tglobal=4096 Tlocal=4096 cp=1 (YARN_SCALE via env) -> probe_run8/csv/ppl_needle_T4096_p1_b2.csv
+[eval] per-position NLL: GPU
+[eval] window 1/8 done
+[eval] window 2/8 done
+[eval] window 3/8 done
+[eval] window 4/8 done
+[eval] window 5/8 done
+[eval] window 6/8 done
+[eval] window 7/8 done
+[eval] window 8/8 done
+[eval] wrote probe_run8/csv/ppl_needle_T4096_p1_b2.csv (4096 positions, 8 windows)

logs/probe_run8/needles/lam_ntk_hd128_th500000_s1.txt

@@ -0,0 +1,4 @@
+n_hat 0
+s 1
+S_search 4096
+lambda 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00 1.00

logs/probe_run8/needles/needle_T4096_absent_b0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb63d258f1d99f6da7541d9be6b7e03ad7fa0924eb231f5317c7a473d1645b7
+size 131076

logs/probe_run8/needles/needle_T4096_absent_b0.json

@@ -0,0 +1,37 @@
+{
+  "bin": "probe_run8/needles/needle_T4096_absent_b0.bin",
+  "variant": "absent",
+  "block": 0,
+  "p_frac": null,
+  "is_null": false,
+  "trials": 8,
+  "seed": 0,
+  "vocab": 50304,
+  "reverse": false,
+  "slot_starts": [
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null
+  ],
+  "T": 4096,
+  "needle_len": 16,
+  "pad_tail": 4,
+  "probe_pos": null,
+  "probe_start": 4076,
+  "answer_start": 4084,
+  "answer_end": 4092,
+  "max_slot_start": 4060,
+  "answer_offsets": [
+    4084,
+    4092
+  ],
+  "csv_positions": [
+    4083,
+    4091
+  ]
+}

logs/probe_run8/needles/needle_T4096_absent_b1.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf9849449af5338369d5a58c1e8df9b431ffbeea7b16ee261af2896438c7f050
+size 131076

logs/probe_run8/needles/needle_T4096_absent_b1.json

@@ -0,0 +1,37 @@
+{
+  "bin": "probe_run8/needles/needle_T4096_absent_b1.bin",
+  "variant": "absent",
+  "block": 1,
+  "p_frac": null,
+  "is_null": false,
+  "trials": 8,
+  "seed": 0,
+  "vocab": 50304,
+  "reverse": false,
+  "slot_starts": [
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null
+  ],
+  "T": 4096,
+  "needle_len": 16,
+  "pad_tail": 4,
+  "probe_pos": null,
+  "probe_start": 4076,
+  "answer_start": 4084,
+  "answer_end": 4092,
+  "max_slot_start": 4060,
+  "answer_offsets": [
+    4084,
+    4092
+  ],
+  "csv_positions": [
+    4083,
+    4091
+  ]
+}

logs/probe_run8/needles/needle_T4096_absent_b2.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99978ca47137c8ade4bb44f3e4979e7dabaaf2a7e2118b399f3eb9291983d455
+size 131076

logs/probe_run8/needles/needle_T4096_absent_b2.json

@@ -0,0 +1,37 @@
+{
+  "bin": "probe_run8/needles/needle_T4096_absent_b2.bin",
+  "variant": "absent",
+  "block": 2,
+  "p_frac": null,
+  "is_null": false,
+  "trials": 8,
+  "seed": 0,
+  "vocab": 50304,
+  "reverse": false,
+  "slot_starts": [
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null
+  ],
+  "T": 4096,
+  "needle_len": 16,
+  "pad_tail": 4,
+  "probe_pos": null,
+  "probe_start": 4076,
+  "answer_start": 4084,
+  "answer_end": 4092,
+  "max_slot_start": 4060,
+  "answer_offsets": [
+    4084,
+    4092
+  ],
+  "csv_positions": [
+    4083,
+    4091
+  ]
+}

logs/probe_run8/needles/needle_T4096_null_b0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3625f3f89f1599ee43b8b2ba0f5d584a6f969d5a3fdcbd679ca47dab111df53
+size 131076