diff --git a/checkpoints/d32cmap/meta_000250.json b/checkpoints/d32cmap/meta_000250.json new file mode 100644 index 0000000000000000000000000000000000000000..488aac54628be34f4c99dfb4ed0b834873ba8fe3 --- /dev/null +++ b/checkpoints/d32cmap/meta_000250.json @@ -0,0 +1,70 @@ +{ + "step": 250, + "val_bpb": 0.8097313191463176, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 3, + "rg_idx": 8, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.8097313191463176, + "smooth_train_loss": 2.700333229256745, + "total_training_time": 598.344233751297 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/meta_000500.json b/checkpoints/d32cmap/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..5185add4259cefdf82c8a244b36909967d7e83d3 --- /dev/null +++ b/checkpoints/d32cmap/meta_000500.json @@ -0,0 +1,70 @@ +{ + "step": 500, + "val_bpb": 0.7586755398685282, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 5, + "rg_idx": 72, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7586755398685282, + "smooth_train_loss": 2.5036690852688066, + "total_training_time": 1224.2108354568481 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/meta_000750.json b/checkpoints/d32cmap/meta_000750.json new file mode 100644 index 0000000000000000000000000000000000000000..43dfd06e62bf6add7424574b380df4ecf5cee4af --- /dev/null +++ b/checkpoints/d32cmap/meta_000750.json @@ -0,0 +1,70 @@ +{ + "step": 750, + "val_bpb": 0.7458365799095863, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 8, + "rg_idx": 56, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7458365799095863, + "smooth_train_loss": 2.58310875916318, + "total_training_time": 1850.6479864120483 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/meta_001000.json b/checkpoints/d32cmap/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..5fcff0adcaf1d1c3ed5b3509dc1157ee3f3e1cd5 --- /dev/null +++ b/checkpoints/d32cmap/meta_001000.json @@ -0,0 +1,70 @@ +{ + "step": 1000, + "val_bpb": 0.7351133002081391, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 11, + "rg_idx": 40, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7351133002081391, + "smooth_train_loss": 2.4455506449672773, + "total_training_time": 2476.7209124565125 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/meta_001250.json b/checkpoints/d32cmap/meta_001250.json new file mode 100644 index 0000000000000000000000000000000000000000..38cb01fbaa892174ed0a2c2ad8a6b830ec7a011f --- /dev/null +++ b/checkpoints/d32cmap/meta_001250.json @@ -0,0 +1,70 @@ +{ + "step": 1250, + "val_bpb": 0.7304292017653182, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 14, + "rg_idx": 24, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7304292017653182, + "smooth_train_loss": 2.3636888406755356, + "total_training_time": 3104.0289120674133 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/meta_001500.json b/checkpoints/d32cmap/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..dcd5b14ee0d761cd9b80335fa6827b67c2895bcd --- /dev/null +++ b/checkpoints/d32cmap/meta_001500.json @@ -0,0 +1,70 @@ +{ + "step": 1500, + "val_bpb": 0.7251844323994228, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 17, + "rg_idx": 8, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7251844323994228, + "smooth_train_loss": 2.3652023803858024, + "total_training_time": 3731.5886714458466 + } +} \ No newline at end of file diff --git a/checkpoints/d32cmap/model_000250.pt b/checkpoints/d32cmap/model_000250.pt new file mode 100644 index 0000000000000000000000000000000000000000..af6864ff67e3ce7584ae649b29762fa96416517f --- /dev/null +++ b/checkpoints/d32cmap/model_000250.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:95a7e29b2260b235cacaf2d6aedb36084fd4c8de7ec13f867f4d3d23e092d454 +size 11542830898 diff --git a/checkpoints/d32cmap/model_000500.pt b/checkpoints/d32cmap/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..6177d7da9dcfef94a9edacf8ef05dff8b54baf49 --- /dev/null +++ b/checkpoints/d32cmap/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab6fdc4b2c5295f481c14e11eaa1dfce3c8d1a8276527140598b4fe25e2b0b93 +size 11542830898 diff --git a/checkpoints/d32cmap/model_000750.pt b/checkpoints/d32cmap/model_000750.pt new file mode 100644 index 0000000000000000000000000000000000000000..a92e86d1ba26a4cd8527eb69d47e3f6badbe61bb --- /dev/null +++ b/checkpoints/d32cmap/model_000750.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1179e3ffdb576496f6dfffe489ad6c8b46cd33fddee4cb12218a48252acf138a +size 11542830898 diff --git a/checkpoints/d32cmap/model_001000.pt b/checkpoints/d32cmap/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f0a28836648e807d15306f194942d0b04e83cb1 --- /dev/null +++ b/checkpoints/d32cmap/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da21a36b776e7b1520509451a72e3de96abb7923792318628fc5813184af1576 +size 11542830898 diff --git a/checkpoints/d32cmap/model_001250.pt b/checkpoints/d32cmap/model_001250.pt new file mode 100644 index 0000000000000000000000000000000000000000..a66bde89880d15f1b1a4282ee3a4d8331cae7c58 --- /dev/null +++ b/checkpoints/d32cmap/model_001250.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f7fd78593f5cc9b70fb582fb8b962dcae688f72e39afdbbca7e46d442c5164 +size 11542830898 diff --git a/checkpoints/d32cmap/model_001500.pt b/checkpoints/d32cmap/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a4caed5a5e6f7383f9f0a068192cb163ab9f12a --- /dev/null +++ b/checkpoints/d32cmap/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cee7edf14e70203b5a66456f917d49370297bbdd6620ff3b361d79f35dbf3fd +size 11542830898 diff --git a/checkpoints/d32cmap/optim_000250_rank0.pt b/checkpoints/d32cmap/optim_000250_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..da4319edd3b1b107856130d7eacdcf9df4624484 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe729274477222ffb8cddbd476581e0e3a20267edeabd74a83b4be2518c8333e +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank1.pt b/checkpoints/d32cmap/optim_000250_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d678082c58a107ccdd5889871f97c04fd78eaec --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f27bc97f8403bd4f5724d5247144209dfdb0a6466f00223978a9815f08f8fd1b +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank2.pt b/checkpoints/d32cmap/optim_000250_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..6b5af19e04f4f77b699ac8e98535082ba4a19007 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:312e896aad5fdc65f7e5ad567f34844aaa19e897cd4a3f2db8e3ffed47e5bd4a +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank3.pt b/checkpoints/d32cmap/optim_000250_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4cff3e2c58ca60c2806bac7b86503017c7134d2 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5924eb05e777dd3778ded375867ca7ed1dc281fe8899b45ca9159ba819e18b5 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank4.pt b/checkpoints/d32cmap/optim_000250_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..29fa668a3ed66ffa384be94625668dea189fd9a2 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24a10e4da6e870f76bbf7b42416f27db9114b2372f10fa86d530cf5ff844578d +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank5.pt b/checkpoints/d32cmap/optim_000250_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..99e1d7b2d206ad34f4c2fea7d01968f64572f361 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c35b00ed1ab556eb62da80ed24220a5f0e1fcb77454784cc9a3ae48e0447a3 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank6.pt b/checkpoints/d32cmap/optim_000250_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..b015436d172ab47a789b5c9fdfffcb2882af5c41 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3e29731451bc9260990ba9165970e3852b331fbf160c607da8a3f1c93a5298cf +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000250_rank7.pt b/checkpoints/d32cmap/optim_000250_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..21d14a4d95d16185858d2c3d9dad4c1cf5a9a2a8 --- /dev/null +++ b/checkpoints/d32cmap/optim_000250_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f1c97adcd2a784278a63e709a86b0c181b11cc116e5c0609066903ec67822ec +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank0.pt b/checkpoints/d32cmap/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..0b0ff7911910d71f0f99d882911712f4110d3123 --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:49a404e5ca46779b54a9ac48db89969370b686c30c2f883b7fe33a2ff7af5c64 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank1.pt b/checkpoints/d32cmap/optim_000500_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..d5ccae27de473536d02203f15f8ba18192969a9e --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3278a5c9875c81311a0c2e790f7ee77ba5fe76e1f6c328fe0391c8bad32a6394 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank2.pt b/checkpoints/d32cmap/optim_000500_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..7a40af0ed2319e6acbfe0fa235c636f709608204 --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6445285147d99ba57eaed41346a0938461756c4a3c0cc8c895a095b097712711 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank3.pt b/checkpoints/d32cmap/optim_000500_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..4763a5945e29f3ba2db3a96f02208754e28d7a01 --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67668a8dfe866ac72b14b4bdd5cd1919aa49c1cbf238170bae0a935b1ddde7ce +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank4.pt b/checkpoints/d32cmap/optim_000500_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..8fca359b7dec69241cd7f52b6c422031e2e49ef4 --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce60030b1d0b120a86edd4185736a68e1aff0422a6620eafdf997f1ca864534d +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank5.pt b/checkpoints/d32cmap/optim_000500_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..58b9ce00a7908be4e0304a05f1c5783eb62f510a --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4caed406b86710b6d0a21cd643fccc7a8f6ef5af7aa8bfa0a059ee564f5899a1 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank6.pt b/checkpoints/d32cmap/optim_000500_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..c2556e2211504940fb489653efd6755b245e837a --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a49042a3f601e4a731832b143c3d202ab36b919e05fd5b9dac447b2044d4ad13 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000500_rank7.pt b/checkpoints/d32cmap/optim_000500_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..348bf5d2561c8bebf822532921a17150be315686 --- /dev/null +++ b/checkpoints/d32cmap/optim_000500_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c857bf2337fe1199f23b1ff1fc5ca3fcee57562ae1fac14c236279e000b2250f +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank0.pt b/checkpoints/d32cmap/optim_000750_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c80db898d4ba77b36ac3f2d45c7e76954788065 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b23903b7111a8c94f913564ed4e073fbea24c216d7103310cb2702c8072174 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank1.pt b/checkpoints/d32cmap/optim_000750_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed5cd8639f5dd06ae778df2106a7bf576c553fbd --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9dfd846589cf1f59d356e2b02bbbd20419c8720f14a1e5448942806897d661 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank2.pt b/checkpoints/d32cmap/optim_000750_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..012dba32364b87437df62c88af4ad72e4ec4cd59 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b714a5ecd2951793d4b1fbd1100a157cf4f4852087343074f98d72dad3e1f600 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank3.pt b/checkpoints/d32cmap/optim_000750_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..206a2e39c42adaca4b618ee51907e8ed170e5eb4 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5291be0628fa2cf019e8ecdf2a78d5e34af05b20754b46a7cefe384c4dffe71e +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank4.pt b/checkpoints/d32cmap/optim_000750_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3ceed10c1c389d46fd5a1687aa5bcb1a6b1e8a9 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71fdedb848dcdac4ed94045e83f60f48661bdf816b0726d2084090f15f5a6b6f +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank5.pt b/checkpoints/d32cmap/optim_000750_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b15e919f9d3a1a9ebd6a409740bb74c7fc15756 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b506e04f143a2a7c97368edd3be01b0d8ba0acc179de8109894f404b913125e9 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank6.pt b/checkpoints/d32cmap/optim_000750_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f65a30fa2453bccfbe9b7b67b84953647e25df8 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce5b481f977e4a3b7dc14c74cfd317a534c62fabf1660f7a05ac988dc307a7d +size 2080789141 diff --git a/checkpoints/d32cmap/optim_000750_rank7.pt b/checkpoints/d32cmap/optim_000750_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..86343fa0a0d31f23df045ad1cbd24781a0490ea1 --- /dev/null +++ b/checkpoints/d32cmap/optim_000750_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa6ad664f07b042e34efc2d459252650e78c07488a356fc5fb15bbc1b599d01d +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank0.pt b/checkpoints/d32cmap/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..7442ff44a40eff01cf894f6565225c55d2f355cb --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d8c8a37a84e5fcca7f37dc942e7ce72c2310e84eb15f613565f90437b0c669 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank1.pt b/checkpoints/d32cmap/optim_001000_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..146e834cb976b413734ead51cef85fab820fb885 --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed8a5e509d127d13866b112305ea16a3d7621e1fdaeddb8598b55c6888eaf8f +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank2.pt b/checkpoints/d32cmap/optim_001000_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea398c2587194cfca8fdc49b20c50b1088a6926d --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8146aa118fcae139d53f9cf65daad8b9dc7284381849fd73580ed1abdaaa5a68 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank3.pt b/checkpoints/d32cmap/optim_001000_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d4592c94be3b11bf09cd11afd60b89388225935 --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9033f4d49037972f96afa0c9271e21d5fa1a7ec17f62fa0efe11eb0582891ea +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank4.pt b/checkpoints/d32cmap/optim_001000_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..74bde67d0ae9a4fafe55d6baa1356d3f73971f5b --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317dcafeb3b278e0fcb10a5723d807040a8492ac4783b2169b83aa76b88e64a8 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank5.pt b/checkpoints/d32cmap/optim_001000_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..682bcb4587bc725ab79485ce12bd3d12a98bce54 --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861b4bf8ea4fa9947e5e5137ca4f6338c87ab7831900ca7d7c7cc21cc972bced +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank6.pt b/checkpoints/d32cmap/optim_001000_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..95079fc4234ab853c718784b23de1c21d4700a32 --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:859f879c4f10ac12b2b3e0f57621cd191b1b360e4accc896b46f0fa2ee0f2ba8 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001000_rank7.pt b/checkpoints/d32cmap/optim_001000_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..d82c83ce2981e472cc4a108feeb945f6e25267cf --- /dev/null +++ b/checkpoints/d32cmap/optim_001000_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9de0f7c282788498a27a031e6304ac504824cfc6a40f3dcdf2abd8b9dcb54d6 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank0.pt b/checkpoints/d32cmap/optim_001250_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5742edff8bc9995d9de607ba32bdb9b092cdb5e --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e49c0887532fdae5def83d1d2f7e0c9b85a5e48ce4258636073216aaf29b7f6 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank1.pt b/checkpoints/d32cmap/optim_001250_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..6907ea5662525c6f5daddda8870ab3feb9a83163 --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e182ec95537f5d185e153439e7dd5564a140245651d4bd933d2ea3ec9b86d7 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank2.pt b/checkpoints/d32cmap/optim_001250_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..7034d82dcd17f4befe264efb4c831cf4cff3324d --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4083d92b6668b23bc26aea477730fcccba85354290b1e8b5ed5d02247b22c0e8 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank3.pt b/checkpoints/d32cmap/optim_001250_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a2719522af87a71d3f0901a64e3783a5d669f9e --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c552a52d25d5cbf2b4f55ede4b6cf35a5f93b2ca264ea01c3da603199616111b +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank4.pt b/checkpoints/d32cmap/optim_001250_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f02bc81ebf865e7f875ae26abe6188c3e12419a --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a151e5374612e0246f35011bf854bcc034666245575dd1020ae93684f99720a +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank5.pt b/checkpoints/d32cmap/optim_001250_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..29c81447db892c93ee24d64483f237ee394a78c6 --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb81c59adbf076e1b0774aec7fbd3e1f56868950a7133bf136201dd286261e28 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank6.pt b/checkpoints/d32cmap/optim_001250_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7ddf40df2d68331b86ac25879fc694862dea02a --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb8419f350e4f4083b2e98343bcbf625286d1eda97f05366c7166325afddc523 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001250_rank7.pt b/checkpoints/d32cmap/optim_001250_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd7ef4642596d0026703f68624a97c5d14c75dd3 --- /dev/null +++ b/checkpoints/d32cmap/optim_001250_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034a924b7458be2ff903990452a56bebe3f76e9995a706c5bf7b79ad2ff688dc +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank0.pt b/checkpoints/d32cmap/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d214dca3576daa06436f0d930d88036eb3cda3ea --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cb3f0ad021fdb64373d2cf483b4bdb74a3e3d708871d426e63434e183f780a +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank1.pt b/checkpoints/d32cmap/optim_001500_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5576c6823a45da3f97726b0f6adf87ddc2f2c548 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a168c836d428b3c25111ac46a5e2584b7d010ef1e1baa505a8f92f896eccea2 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank2.pt b/checkpoints/d32cmap/optim_001500_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1132052c50234826b1d4c6f46a52c33aef67ffd --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314e12d044293ffeb4dc87269b3808cb36c59e584f34ae4293ad7ad48dedb2d2 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank3.pt b/checkpoints/d32cmap/optim_001500_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..154fef95681623e55c53db9bac56b89ea1e41c62 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2ad7d976dbb321127c62e9813ee6e3fbe2db659d120620a69a3a20504492cb +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank4.pt b/checkpoints/d32cmap/optim_001500_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..94dfc7d76756e5ef52cab8b101d8591183631dd6 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5edf02538782e38d400b2b5e91a421d48b1fa1a10e3af00b4eb09b429c212150 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank5.pt b/checkpoints/d32cmap/optim_001500_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..65744c72f512cb4c218e062485c5ced1e20d8ab1 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61769e89dc3b8dc8898b3059aa861e6c98f55d1e76a80fca09381fc28ce5db78 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank6.pt b/checkpoints/d32cmap/optim_001500_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..390dcecbceefb97fc8e52861b1b05a2a58b4ce58 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516c564e0b5c51134264df1b66cd26f3402c7ab344f64b58d1f56aacccf2b244 +size 2080789141 diff --git a/checkpoints/d32cmap/optim_001500_rank7.pt b/checkpoints/d32cmap/optim_001500_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5c7371f0fb15e20786e2a6b05099bf727f8c439 --- /dev/null +++ b/checkpoints/d32cmap/optim_001500_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee248c1485cdd59eff8af5e18c3c19a6eaf202637013b93becdeb5fe51c2f59 +size 2080789141 diff --git a/latest/meta_000750.json b/latest/meta_000750.json new file mode 100644 index 0000000000000000000000000000000000000000..43dfd06e62bf6add7424574b380df4ecf5cee4af --- /dev/null +++ b/latest/meta_000750.json @@ -0,0 +1,70 @@ +{ + "step": 750, + "val_bpb": 0.7458365799095863, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 8, + "rg_idx": 56, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7458365799095863, + "smooth_train_loss": 2.58310875916318, + "total_training_time": 1850.6479864120483 + } +} \ No newline at end of file diff --git a/latest/meta_001000.json b/latest/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..5fcff0adcaf1d1c3ed5b3509dc1157ee3f3e1cd5 --- /dev/null +++ b/latest/meta_001000.json @@ -0,0 +1,70 @@ +{ + "step": 1000, + "val_bpb": 0.7351133002081391, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 11, + "rg_idx": 40, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7351133002081391, + "smooth_train_loss": 2.4455506449672773, + "total_training_time": 2476.7209124565125 + } +} \ No newline at end of file diff --git a/latest/meta_001250.json b/latest/meta_001250.json new file mode 100644 index 0000000000000000000000000000000000000000..38cb01fbaa892174ed0a2c2ad8a6b830ec7a011f --- /dev/null +++ b/latest/meta_001250.json @@ -0,0 +1,70 @@ +{ + "step": 1250, + "val_bpb": 0.7304292017653182, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 14, + "rg_idx": 24, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7304292017653182, + "smooth_train_loss": 2.3636888406755356, + "total_training_time": 3104.0289120674133 + } +} \ No newline at end of file diff --git a/latest/meta_001500.json b/latest/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..dcd5b14ee0d761cd9b80335fa6827b67c2895bcd --- /dev/null +++ b/latest/meta_001500.json @@ -0,0 +1,70 @@ +{ + "step": 1500, + "val_bpb": 0.7251844323994228, + "model_config": { + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false + }, + "user_config": { + "run": "dummy", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 32, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "objective": "ar", + "mlm_mask_prob": 0.15, + "num_iterations": 1500, + "target_flops": -1.0, + "target_param_data_ratio": 12, + "device_batch_size": 2, + "total_batch_size": 524288, + "embedding_lr": 0.03, + "unembedding_lr": 0.0008, + "weight_decay": 0.0, + "matrix_lr": 0.002, + "scalar_lr": 0.05, + "warmup_steps": 40, + "warmdown_ratio": 0.0, + "final_lr_frac": 1.0, + "resume_from_step": -1, + "init_from_model": "/root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt", + "eval_every": 100, + "eval_tokens": 41943040, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": 250, + "save_every": 250, + "model_tag": "d32cmap" + }, + "device_batch_size": 2, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "pq_idx": 17, + "rg_idx": 8, + "epoch": 1 + }, + "loop_state": { + "min_val_bpb": 0.7251844323994228, + "smooth_train_loss": 2.3652023803858024, + "total_training_time": 3731.5886714458466 + } +} \ No newline at end of file diff --git a/latest/model_000750.pt b/latest/model_000750.pt new file mode 100644 index 0000000000000000000000000000000000000000..a92e86d1ba26a4cd8527eb69d47e3f6badbe61bb --- /dev/null +++ b/latest/model_000750.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1179e3ffdb576496f6dfffe489ad6c8b46cd33fddee4cb12218a48252acf138a +size 11542830898 diff --git a/latest/model_001000.pt b/latest/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f0a28836648e807d15306f194942d0b04e83cb1 --- /dev/null +++ b/latest/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da21a36b776e7b1520509451a72e3de96abb7923792318628fc5813184af1576 +size 11542830898 diff --git a/latest/model_001250.pt b/latest/model_001250.pt new file mode 100644 index 0000000000000000000000000000000000000000..a66bde89880d15f1b1a4282ee3a4d8331cae7c58 --- /dev/null +++ b/latest/model_001250.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74f7fd78593f5cc9b70fb582fb8b962dcae688f72e39afdbbca7e46d442c5164 +size 11542830898 diff --git a/latest/model_001500.pt b/latest/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a4caed5a5e6f7383f9f0a068192cb163ab9f12a --- /dev/null +++ b/latest/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cee7edf14e70203b5a66456f917d49370297bbdd6620ff3b361d79f35dbf3fd +size 11542830898 diff --git a/latest/optim_000750_rank0.pt b/latest/optim_000750_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c80db898d4ba77b36ac3f2d45c7e76954788065 --- /dev/null +++ b/latest/optim_000750_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53b23903b7111a8c94f913564ed4e073fbea24c216d7103310cb2702c8072174 +size 2080789141 diff --git a/latest/optim_000750_rank1.pt b/latest/optim_000750_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed5cd8639f5dd06ae778df2106a7bf576c553fbd --- /dev/null +++ b/latest/optim_000750_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a9dfd846589cf1f59d356e2b02bbbd20419c8720f14a1e5448942806897d661 +size 2080789141 diff --git a/latest/optim_000750_rank2.pt b/latest/optim_000750_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..012dba32364b87437df62c88af4ad72e4ec4cd59 --- /dev/null +++ b/latest/optim_000750_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b714a5ecd2951793d4b1fbd1100a157cf4f4852087343074f98d72dad3e1f600 +size 2080789141 diff --git a/latest/optim_000750_rank3.pt b/latest/optim_000750_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..206a2e39c42adaca4b618ee51907e8ed170e5eb4 --- /dev/null +++ b/latest/optim_000750_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5291be0628fa2cf019e8ecdf2a78d5e34af05b20754b46a7cefe384c4dffe71e +size 2080789141 diff --git a/latest/optim_000750_rank4.pt b/latest/optim_000750_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..d3ceed10c1c389d46fd5a1687aa5bcb1a6b1e8a9 --- /dev/null +++ b/latest/optim_000750_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71fdedb848dcdac4ed94045e83f60f48661bdf816b0726d2084090f15f5a6b6f +size 2080789141 diff --git a/latest/optim_000750_rank5.pt b/latest/optim_000750_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b15e919f9d3a1a9ebd6a409740bb74c7fc15756 --- /dev/null +++ b/latest/optim_000750_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b506e04f143a2a7c97368edd3be01b0d8ba0acc179de8109894f404b913125e9 +size 2080789141 diff --git a/latest/optim_000750_rank6.pt b/latest/optim_000750_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f65a30fa2453bccfbe9b7b67b84953647e25df8 --- /dev/null +++ b/latest/optim_000750_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2ce5b481f977e4a3b7dc14c74cfd317a534c62fabf1660f7a05ac988dc307a7d +size 2080789141 diff --git a/latest/optim_000750_rank7.pt b/latest/optim_000750_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..86343fa0a0d31f23df045ad1cbd24781a0490ea1 --- /dev/null +++ b/latest/optim_000750_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa6ad664f07b042e34efc2d459252650e78c07488a356fc5fb15bbc1b599d01d +size 2080789141 diff --git a/latest/optim_001000_rank0.pt b/latest/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..7442ff44a40eff01cf894f6565225c55d2f355cb --- /dev/null +++ b/latest/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39d8c8a37a84e5fcca7f37dc942e7ce72c2310e84eb15f613565f90437b0c669 +size 2080789141 diff --git a/latest/optim_001000_rank1.pt b/latest/optim_001000_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..146e834cb976b413734ead51cef85fab820fb885 --- /dev/null +++ b/latest/optim_001000_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aed8a5e509d127d13866b112305ea16a3d7621e1fdaeddb8598b55c6888eaf8f +size 2080789141 diff --git a/latest/optim_001000_rank2.pt b/latest/optim_001000_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea398c2587194cfca8fdc49b20c50b1088a6926d --- /dev/null +++ b/latest/optim_001000_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8146aa118fcae139d53f9cf65daad8b9dc7284381849fd73580ed1abdaaa5a68 +size 2080789141 diff --git a/latest/optim_001000_rank3.pt b/latest/optim_001000_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d4592c94be3b11bf09cd11afd60b89388225935 --- /dev/null +++ b/latest/optim_001000_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9033f4d49037972f96afa0c9271e21d5fa1a7ec17f62fa0efe11eb0582891ea +size 2080789141 diff --git a/latest/optim_001000_rank4.pt b/latest/optim_001000_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..74bde67d0ae9a4fafe55d6baa1356d3f73971f5b --- /dev/null +++ b/latest/optim_001000_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:317dcafeb3b278e0fcb10a5723d807040a8492ac4783b2169b83aa76b88e64a8 +size 2080789141 diff --git a/latest/optim_001000_rank5.pt b/latest/optim_001000_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..682bcb4587bc725ab79485ce12bd3d12a98bce54 --- /dev/null +++ b/latest/optim_001000_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:861b4bf8ea4fa9947e5e5137ca4f6338c87ab7831900ca7d7c7cc21cc972bced +size 2080789141 diff --git a/latest/optim_001000_rank6.pt b/latest/optim_001000_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..95079fc4234ab853c718784b23de1c21d4700a32 --- /dev/null +++ b/latest/optim_001000_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:859f879c4f10ac12b2b3e0f57621cd191b1b360e4accc896b46f0fa2ee0f2ba8 +size 2080789141 diff --git a/latest/optim_001000_rank7.pt b/latest/optim_001000_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..d82c83ce2981e472cc4a108feeb945f6e25267cf --- /dev/null +++ b/latest/optim_001000_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9de0f7c282788498a27a031e6304ac504824cfc6a40f3dcdf2abd8b9dcb54d6 +size 2080789141 diff --git a/latest/optim_001250_rank0.pt b/latest/optim_001250_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c5742edff8bc9995d9de607ba32bdb9b092cdb5e --- /dev/null +++ b/latest/optim_001250_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e49c0887532fdae5def83d1d2f7e0c9b85a5e48ce4258636073216aaf29b7f6 +size 2080789141 diff --git a/latest/optim_001250_rank1.pt b/latest/optim_001250_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..6907ea5662525c6f5daddda8870ab3feb9a83163 --- /dev/null +++ b/latest/optim_001250_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f9e182ec95537f5d185e153439e7dd5564a140245651d4bd933d2ea3ec9b86d7 +size 2080789141 diff --git a/latest/optim_001250_rank2.pt b/latest/optim_001250_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..7034d82dcd17f4befe264efb4c831cf4cff3324d --- /dev/null +++ b/latest/optim_001250_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4083d92b6668b23bc26aea477730fcccba85354290b1e8b5ed5d02247b22c0e8 +size 2080789141 diff --git a/latest/optim_001250_rank3.pt b/latest/optim_001250_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..8a2719522af87a71d3f0901a64e3783a5d669f9e --- /dev/null +++ b/latest/optim_001250_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c552a52d25d5cbf2b4f55ede4b6cf35a5f93b2ca264ea01c3da603199616111b +size 2080789141 diff --git a/latest/optim_001250_rank4.pt b/latest/optim_001250_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..4f02bc81ebf865e7f875ae26abe6188c3e12419a --- /dev/null +++ b/latest/optim_001250_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a151e5374612e0246f35011bf854bcc034666245575dd1020ae93684f99720a +size 2080789141 diff --git a/latest/optim_001250_rank5.pt b/latest/optim_001250_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..29c81447db892c93ee24d64483f237ee394a78c6 --- /dev/null +++ b/latest/optim_001250_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb81c59adbf076e1b0774aec7fbd3e1f56868950a7133bf136201dd286261e28 +size 2080789141 diff --git a/latest/optim_001250_rank6.pt b/latest/optim_001250_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7ddf40df2d68331b86ac25879fc694862dea02a --- /dev/null +++ b/latest/optim_001250_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb8419f350e4f4083b2e98343bcbf625286d1eda97f05366c7166325afddc523 +size 2080789141 diff --git a/latest/optim_001250_rank7.pt b/latest/optim_001250_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd7ef4642596d0026703f68624a97c5d14c75dd3 --- /dev/null +++ b/latest/optim_001250_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:034a924b7458be2ff903990452a56bebe3f76e9995a706c5bf7b79ad2ff688dc +size 2080789141 diff --git a/latest/optim_001500_rank0.pt b/latest/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..d214dca3576daa06436f0d930d88036eb3cda3ea --- /dev/null +++ b/latest/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23cb3f0ad021fdb64373d2cf483b4bdb74a3e3d708871d426e63434e183f780a +size 2080789141 diff --git a/latest/optim_001500_rank1.pt b/latest/optim_001500_rank1.pt new file mode 100644 index 0000000000000000000000000000000000000000..5576c6823a45da3f97726b0f6adf87ddc2f2c548 --- /dev/null +++ b/latest/optim_001500_rank1.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a168c836d428b3c25111ac46a5e2584b7d010ef1e1baa505a8f92f896eccea2 +size 2080789141 diff --git a/latest/optim_001500_rank2.pt b/latest/optim_001500_rank2.pt new file mode 100644 index 0000000000000000000000000000000000000000..e1132052c50234826b1d4c6f46a52c33aef67ffd --- /dev/null +++ b/latest/optim_001500_rank2.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:314e12d044293ffeb4dc87269b3808cb36c59e584f34ae4293ad7ad48dedb2d2 +size 2080789141 diff --git a/latest/optim_001500_rank3.pt b/latest/optim_001500_rank3.pt new file mode 100644 index 0000000000000000000000000000000000000000..154fef95681623e55c53db9bac56b89ea1e41c62 --- /dev/null +++ b/latest/optim_001500_rank3.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2ad7d976dbb321127c62e9813ee6e3fbe2db659d120620a69a3a20504492cb +size 2080789141 diff --git a/latest/optim_001500_rank4.pt b/latest/optim_001500_rank4.pt new file mode 100644 index 0000000000000000000000000000000000000000..94dfc7d76756e5ef52cab8b101d8591183631dd6 --- /dev/null +++ b/latest/optim_001500_rank4.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5edf02538782e38d400b2b5e91a421d48b1fa1a10e3af00b4eb09b429c212150 +size 2080789141 diff --git a/latest/optim_001500_rank5.pt b/latest/optim_001500_rank5.pt new file mode 100644 index 0000000000000000000000000000000000000000..65744c72f512cb4c218e062485c5ced1e20d8ab1 --- /dev/null +++ b/latest/optim_001500_rank5.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61769e89dc3b8dc8898b3059aa861e6c98f55d1e76a80fca09381fc28ce5db78 +size 2080789141 diff --git a/latest/optim_001500_rank6.pt b/latest/optim_001500_rank6.pt new file mode 100644 index 0000000000000000000000000000000000000000..390dcecbceefb97fc8e52861b1b05a2a58b4ce58 --- /dev/null +++ b/latest/optim_001500_rank6.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:516c564e0b5c51134264df1b66cd26f3402c7ab344f64b58d1f56aacccf2b244 +size 2080789141 diff --git a/latest/optim_001500_rank7.pt b/latest/optim_001500_rank7.pt new file mode 100644 index 0000000000000000000000000000000000000000..b5c7371f0fb15e20786e2a6b05099bf727f8c439 --- /dev/null +++ b/latest/optim_001500_rank7.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee248c1485cdd59eff8af5e18c3c19a6eaf202637013b93becdeb5fe51c2f59 +size 2080789141 diff --git a/logs/d32cmap_icl.csv b/logs/d32cmap_icl.csv new file mode 100644 index 0000000000000000000000000000000000000000..8fca8f810789c315fd187d82584190c98d5c7200 --- /dev/null +++ b/logs/d32cmap_icl.csv @@ -0,0 +1,17 @@ +step,icl_early,icl_late,icl_score,induction_loss,induction_acc,random_half_loss +0,5.1304,5.8076,0.6772,15.0148,0.0,15.1274 +100,3.2338,3.3452,0.1114,13.4303,0.0038,14.8965 +200,3.0344,3.0982,0.0638,12.2065,0.0161,14.5413 +300,2.933,3.0051,0.072,11.5097,0.0289,14.3222 +400,2.8437,2.9914,0.1477,10.8087,0.0461,14.1634 +500,2.784,2.9436,0.1596,10.402,0.0576,14.055 +600,2.7678,2.9164,0.1486,10.136,0.0637,13.993 +700,2.7512,2.9013,0.1501,9.7585,0.0784,13.9678 +800,2.7298,2.8802,0.1504,9.6485,0.0788,13.9446 +900,2.7088,2.8737,0.1649,9.3294,0.0943,13.9594 +1000,2.678,2.865,0.187,9.4661,0.0879,14.006 +1100,2.6734,2.8559,0.1825,9.4218,0.0877,13.9912 +1200,2.6633,2.8423,0.179,9.2954,0.0939,14.1187 +1300,2.6509,2.8619,0.211,9.3552,0.0929,14.1283 +1400,2.6447,2.8352,0.1905,9.4753,0.0817,14.0049 +1500,2.6255,2.819,0.1936,9.1172,0.101,14.1163 diff --git a/logs/d32cmap_metrics.csv b/logs/d32cmap_metrics.csv new file mode 100644 index 0000000000000000000000000000000000000000..649c4e68580ec72ffda302ef83e48ce535dfe254 --- /dev/null +++ b/logs/d32cmap_metrics.csv @@ -0,0 +1,1501 @@ +step,total_steps,loss,lrm,dt_ms,tok_per_sec,bf16_mfu +0,1500,5.939279,0.03,173819.1,3016,0.46 +1,1500,5.956664,0.05,2685.29,195244,29.81 +2,1500,5.88379,0.07,2861.3,183234,27.98 +3,1500,5.876734,0.1,2482.99,211152,32.24 +4,1500,5.805432,0.12,2494.62,210167,32.09 +5,1500,5.792202,0.15,2494.67,210162,32.09 +6,1500,5.767968,0.17,2502.11,209538,31.99 +7,1500,5.776086,0.2,2500.57,209667,32.01 +8,1500,5.7634,0.23,2490.52,210513,32.14 +9,1500,5.676017,0.25,2494.58,210170,32.09 +10,1500,5.621683,0.28,2497.56,209919,32.05 +11,1500,5.571587,0.3,2499.24,209779,32.03 +12,1500,5.548217,0.33,2498.47,209843,32.04 +13,1500,5.470431,0.35,2498.29,209858,32.04 +14,1500,5.43045,0.38,2501.25,209610,32.0 +15,1500,5.347769,0.4,2499.86,209726,32.02 +16,1500,5.246905,0.42,2499.73,209737,32.02 +17,1500,5.17243,0.45,2493.68,210246,32.1 +18,1500,5.120843,0.47,2499.59,209749,32.02 +19,1500,5.091685,0.5,2494.79,210153,32.09 +20,1500,5.031218,0.53,2497.78,209901,32.05 +21,1500,4.924438,0.55,2500.8,209648,32.01 +22,1500,4.878394,0.57,2500.97,209633,32.01 +23,1500,4.790773,0.6,2500.81,209647,32.01 +24,1500,4.75017,0.62,2502.89,209473,31.98 +25,1500,4.698532,0.65,2500.15,209703,32.02 +26,1500,4.655308,0.68,2497.32,209939,32.05 +27,1500,4.655427,0.7,2501.75,209568,32.0 +28,1500,4.569183,0.72,2500.24,209695,32.02 +29,1500,4.494453,0.75,2495.72,210074,32.07 +30,1500,4.472533,0.78,2500.82,209646,32.01 +31,1500,4.447625,0.8,2502.08,209541,31.99 +32,1500,4.363409,0.82,2502.24,209527,31.99 +33,1500,4.322013,0.85,2499.29,209775,32.03 +34,1500,4.241499,0.88,2501.64,209577,32.0 +35,1500,4.149794,0.9,2498.59,209833,32.04 +36,1500,4.086062,0.93,2500.66,209659,32.01 +37,1500,4.060564,0.95,2495.85,210063,32.07 +38,1500,4.015661,0.97,2498.2,209866,32.04 +39,1500,3.966935,1.0,2496.4,210017,32.06 +40,1500,3.943025,1.0,2495.8,210067,32.07 +41,1500,3.92166,1.0,2504.6,209329,31.96 +42,1500,3.91871,1.0,2497.4,209933,32.05 +43,1500,3.887281,1.0,2502.83,209477,31.98 +44,1500,3.897078,1.0,2499.14,209787,32.03 +45,1500,3.82448,1.0,2500.7,209656,32.01 +46,1500,3.72153,1.0,2497.85,209895,32.05 +47,1500,3.68474,1.0,2500.95,209635,32.01 +48,1500,3.662864,1.0,2497.91,209890,32.04 +49,1500,3.596936,1.0,2497.21,209949,32.05 +50,1500,3.584375,1.0,2501.48,209591,32.0 +51,1500,3.537943,1.0,2501.91,209555,31.99 +52,1500,3.532923,1.0,2494.04,210216,32.09 +53,1500,3.510258,1.0,2500.43,209679,32.01 +54,1500,3.510891,1.0,2497.17,209952,32.05 +55,1500,3.529868,1.0,2499.21,209781,32.03 +56,1500,3.606472,1.0,2502.32,209521,31.99 +57,1500,3.568959,1.0,2497.56,209919,32.05 +58,1500,3.569934,1.0,2496.95,209971,32.06 +59,1500,3.554943,1.0,2496.78,209985,32.06 +60,1500,3.484738,1.0,2496.9,209975,32.06 +61,1500,3.424122,1.0,2496.32,210024,32.07 +62,1500,3.383661,1.0,2502.71,209488,31.98 +63,1500,3.327162,1.0,2498.35,209854,32.04 +64,1500,3.325502,1.0,2501.26,209609,32.0 +65,1500,3.30794,1.0,2499.18,209783,32.03 +66,1500,3.332836,1.0,2500.05,209711,32.02 +67,1500,3.303473,1.0,2494.88,210145,32.08 +68,1500,3.301317,1.0,2499.16,209785,32.03 +69,1500,3.300453,1.0,2497.43,209931,32.05 +70,1500,3.303051,1.0,2503.44,209427,31.97 +71,1500,3.346494,1.0,2647.3,198046,30.24 +72,1500,3.331093,1.0,2501.62,209579,32.0 +73,1500,3.307411,1.0,2497.78,209901,32.05 +74,1500,3.285279,1.0,2502.72,209486,31.98 +75,1500,3.246838,1.0,2496.13,210040,32.07 +76,1500,3.204915,1.0,2496.94,209972,32.06 +77,1500,3.17397,1.0,2501.91,209555,31.99 +78,1500,3.154224,1.0,2505.64,209243,31.95 +79,1500,3.160177,1.0,2500.09,209707,32.02 +80,1500,3.208149,1.0,2505.87,209224,31.94 +81,1500,3.227993,1.0,2497.36,209936,32.05 +82,1500,3.244446,1.0,2503.82,209395,31.97 +83,1500,3.228024,1.0,2497.13,209956,32.05 +84,1500,3.237289,1.0,2495.61,210084,32.07 +85,1500,3.227059,1.0,2499.86,209727,32.02 +86,1500,3.235147,1.0,2498.19,209866,32.04 +87,1500,3.183798,1.0,2497.28,209943,32.05 +88,1500,3.160538,1.0,2500.68,209658,32.01 +89,1500,3.142993,1.0,2493.69,210245,32.1 +90,1500,3.148296,1.0,2500.75,209652,32.01 +91,1500,3.14894,1.0,2497.71,209907,32.05 +92,1500,3.138557,1.0,2495.0,210135,32.08 +93,1500,3.047582,1.0,2499.25,209777,32.03 +94,1500,3.023392,1.0,2498.33,209855,32.04 +95,1500,2.988202,1.0,2499.87,209725,32.02 +96,1500,2.978244,1.0,2502.08,209540,31.99 +97,1500,2.94374,1.0,2498.33,209855,32.04 +98,1500,2.956456,1.0,2501.04,209628,32.0 +99,1500,2.91794,1.0,2500.95,209635,32.01 +100,1500,2.915524,1.0,2497.35,209937,32.05 +101,1500,2.898875,1.0,2496.59,210001,32.06 +102,1500,2.925061,1.0,2505.95,209217,31.94 +103,1500,2.932929,1.0,2496.37,210020,32.06 +104,1500,2.906146,1.0,2497.89,209892,32.05 +105,1500,2.907918,1.0,2502.32,209520,31.99 +106,1500,2.924005,1.0,2506.34,209184,31.94 +107,1500,2.904355,1.0,2505.2,209279,31.95 +108,1500,2.956387,1.0,2500.24,209695,32.02 +109,1500,2.921634,1.0,2498.37,209852,32.04 +110,1500,2.92177,1.0,2498.71,209823,32.03 +111,1500,2.914309,1.0,2499.01,209798,32.03 +112,1500,2.934415,1.0,2499.8,209731,32.02 +113,1500,2.912347,1.0,2499.1,209790,32.03 +114,1500,2.897675,1.0,2498.62,209830,32.04 +115,1500,2.918555,1.0,2496.98,209968,32.06 +116,1500,2.92153,1.0,2502.7,209489,31.98 +117,1500,2.926781,1.0,2501.53,209586,32.0 +118,1500,2.944181,1.0,2503.88,209390,31.97 +119,1500,2.963816,1.0,2494.88,210145,32.08 +120,1500,2.978842,1.0,2499.18,209784,32.03 +121,1500,2.982057,1.0,2500.73,209653,32.01 +122,1500,2.973189,1.0,2500.84,209644,32.01 +123,1500,2.962652,1.0,2499.48,209758,32.02 +124,1500,2.924283,1.0,2503.58,209415,31.97 +125,1500,2.920081,1.0,2501.28,209607,32.0 +126,1500,2.920181,1.0,2500.93,209637,32.01 +127,1500,2.946619,1.0,2500.6,209665,32.01 +128,1500,2.953652,1.0,2501.46,209592,32.0 +129,1500,2.943835,1.0,2497.84,209896,32.05 +130,1500,2.915199,1.0,2508.27,209023,31.91 +131,1500,2.83593,1.0,2498.84,209812,32.03 +132,1500,2.796642,1.0,2500.51,209672,32.01 +133,1500,2.824124,1.0,2496.34,210022,32.07 +134,1500,2.820834,1.0,2502.11,209538,31.99 +135,1500,2.827388,1.0,2504.91,209304,31.96 +136,1500,2.829499,1.0,2501.8,209564,32.0 +137,1500,2.877256,1.0,2499.21,209781,32.03 +138,1500,2.871995,1.0,2499.98,209717,32.02 +139,1500,2.857608,1.0,2498.79,209817,32.03 +140,1500,2.870178,1.0,2502.12,209537,31.99 +141,1500,2.891083,1.0,2502.49,209506,31.99 +142,1500,2.93861,1.0,2501.14,209619,32.0 +143,1500,2.966213,1.0,2498.9,209807,32.03 +144,1500,2.97514,1.0,2498.56,209835,32.04 +145,1500,2.964519,1.0,2749.59,190678,29.11 +146,1500,2.953216,1.0,2499.9,209723,32.02 +147,1500,2.914044,1.0,2500.78,209649,32.01 +148,1500,2.920703,1.0,2499.11,209789,32.03 +149,1500,2.899524,1.0,2498.52,209839,32.04 +150,1500,2.917274,1.0,2502.08,209540,31.99 +151,1500,2.939549,1.0,2498.4,209849,32.04 +152,1500,2.917839,1.0,2499.72,209738,32.02 +153,1500,2.856035,1.0,2500.77,209650,32.01 +154,1500,2.805288,1.0,2503.66,209408,31.97 +155,1500,2.785939,1.0,2501.78,209566,32.0 +156,1500,2.786126,1.0,2501.44,209594,32.0 +157,1500,2.739186,1.0,2501.45,209593,32.0 +158,1500,2.744949,1.0,2501.34,209603,32.0 +159,1500,2.774082,1.0,2498.28,209859,32.04 +160,1500,2.714753,1.0,2510.83,208810,31.88 +161,1500,2.688827,1.0,2498.04,209879,32.04 +162,1500,2.656541,1.0,2501.99,209548,31.99 +163,1500,2.671044,1.0,2506.87,209140,31.93 +164,1500,2.712461,1.0,2503.64,209410,31.97 +165,1500,2.73072,1.0,2504.83,209310,31.96 +166,1500,2.751532,1.0,2500.21,209697,32.02 +167,1500,2.798152,1.0,2501.45,209593,32.0 +168,1500,2.865066,1.0,2502.38,209515,31.99 +169,1500,2.86098,1.0,2504.08,209373,31.97 +170,1500,2.853965,1.0,2497.51,209924,32.05 +171,1500,2.88235,1.0,2503.4,209430,31.97 +172,1500,2.881416,1.0,2501.18,209616,32.0 +173,1500,2.835252,1.0,2500.53,209670,32.01 +174,1500,2.866169,1.0,2499.69,209741,32.02 +175,1500,2.88437,1.0,2500.46,209676,32.01 +176,1500,2.859987,1.0,2504.46,209341,31.96 +177,1500,2.856667,1.0,2503.79,209397,31.97 +178,1500,2.856164,1.0,2504.08,209373,31.97 +179,1500,2.863707,1.0,2501.25,209610,32.0 +180,1500,2.83386,1.0,2499.77,209734,32.02 +181,1500,2.809028,1.0,2500.77,209650,32.01 +182,1500,2.818415,1.0,2499.14,209787,32.03 +183,1500,2.826522,1.0,2502.62,209495,31.98 +184,1500,2.807088,1.0,2502.9,209472,31.98 +185,1500,2.785449,1.0,2498.21,209865,32.04 +186,1500,2.777982,1.0,2502.86,209475,31.98 +187,1500,2.793644,1.0,2499.91,209722,32.02 +188,1500,2.772648,1.0,2500.69,209656,32.01 +189,1500,2.773812,1.0,2505.01,209295,31.95 +190,1500,2.774943,1.0,2503.36,209433,31.98 +191,1500,2.763321,1.0,2500.16,209701,32.02 +192,1500,2.72871,1.0,2502.47,209508,31.99 +193,1500,2.780058,1.0,2499.87,209725,32.02 +194,1500,2.777232,1.0,2500.12,209705,32.02 +195,1500,2.808145,1.0,2500.63,209662,32.01 +196,1500,2.792706,1.0,2497.46,209928,32.05 +197,1500,2.789577,1.0,2500.91,209638,32.01 +198,1500,2.842831,1.0,2499.25,209777,32.03 +199,1500,2.854176,1.0,2496.81,209983,32.06 +200,1500,2.840186,1.0,2499.73,209738,32.02 +201,1500,2.777129,1.0,2497.93,209888,32.04 +202,1500,2.757498,1.0,2500.63,209662,32.01 +203,1500,2.731559,1.0,2501.13,209620,32.0 +204,1500,2.736987,1.0,2499.95,209719,32.02 +205,1500,2.719595,1.0,2495.74,210073,32.07 +206,1500,2.702856,1.0,2503.57,209416,31.97 +207,1500,2.7431,1.0,2500.33,209687,32.01 +208,1500,2.731707,1.0,2498.31,209857,32.04 +209,1500,2.750448,1.0,2505.58,209248,31.95 +210,1500,2.651502,1.0,2498.41,209848,32.04 +211,1500,2.643204,1.0,2507.21,209112,31.93 +212,1500,2.63658,1.0,2502.34,209518,31.99 +213,1500,2.646286,1.0,2501.99,209548,31.99 +214,1500,2.657232,1.0,2501.48,209591,32.0 +215,1500,2.670429,1.0,2499.4,209765,32.03 +216,1500,2.685346,1.0,2502.41,209513,31.99 +217,1500,2.687898,1.0,2503.66,209408,31.97 +218,1500,2.686027,1.0,2499.84,209728,32.02 +219,1500,2.746372,1.0,2905.22,180464,27.55 +220,1500,2.746497,1.0,2499.56,209752,32.02 +221,1500,2.755573,1.0,2504.56,209333,31.96 +222,1500,2.775914,1.0,2496.1,210043,32.07 +223,1500,2.712128,1.0,2497.29,209942,32.05 +224,1500,2.699746,1.0,2504.78,209314,31.96 +225,1500,2.695812,1.0,2500.45,209677,32.01 +226,1500,2.651124,1.0,2501.0,209631,32.01 +227,1500,2.630231,1.0,2496.61,210000,32.06 +228,1500,2.631078,1.0,2503.45,209425,31.97 +229,1500,2.615961,1.0,2503.66,209408,31.97 +230,1500,2.634345,1.0,2494.33,210191,32.09 +231,1500,2.658159,1.0,2498.41,209848,32.04 +232,1500,2.665212,1.0,2499.67,209743,32.02 +233,1500,2.702525,1.0,2499.89,209724,32.02 +234,1500,2.688055,1.0,2498.0,209883,32.04 +235,1500,2.677379,1.0,2500.49,209674,32.01 +236,1500,2.66552,1.0,2499.24,209778,32.03 +237,1500,2.635362,1.0,2498.0,209883,32.04 +238,1500,2.681989,1.0,2500.71,209656,32.01 +239,1500,2.665659,1.0,2497.48,209926,32.05 +240,1500,2.653508,1.0,2498.78,209817,32.03 +241,1500,2.631025,1.0,2498.98,209800,32.03 +242,1500,2.629892,1.0,2498.83,209813,32.03 +243,1500,2.6424,1.0,2498.98,209800,32.03 +244,1500,2.66473,1.0,2497.75,209903,32.05 +245,1500,2.677625,1.0,2496.7,209992,32.06 +246,1500,2.61927,1.0,2499.02,209797,32.03 +247,1500,2.646338,1.0,2500.25,209693,32.01 +248,1500,2.684204,1.0,2505.23,209277,31.95 +249,1500,2.700333,1.0,2505.08,209290,31.95 +250,1500,2.70105,1.0,2475.05,211828,32.34 +251,1500,2.69439,1.0,2482.94,211156,32.24 +252,1500,2.676868,1.0,2497.56,209920,32.05 +253,1500,2.67216,1.0,2491.61,210421,32.13 +254,1500,2.669649,1.0,2496.71,209991,32.06 +255,1500,2.682889,1.0,2498.7,209823,32.03 +256,1500,2.690229,1.0,2500.71,209655,32.01 +257,1500,2.713785,1.0,2500.17,209701,32.02 +258,1500,2.744359,1.0,2495.7,210076,32.07 +259,1500,2.716034,1.0,2494.01,210219,32.1 +260,1500,2.699451,1.0,2492.83,210318,32.11 +261,1500,2.692689,1.0,2498.95,209803,32.03 +262,1500,2.710181,1.0,2498.71,209823,32.03 +263,1500,2.706901,1.0,2495.49,210093,32.08 +264,1500,2.721221,1.0,2496.82,209981,32.06 +265,1500,2.756175,1.0,2499.27,209776,32.03 +266,1500,2.752914,1.0,2495.08,210129,32.08 +267,1500,2.741282,1.0,2497.5,209925,32.05 +268,1500,2.730078,1.0,2498.35,209853,32.04 +269,1500,2.656281,1.0,2490.78,210491,32.14 +270,1500,2.609942,1.0,2491.76,210408,32.12 +271,1500,2.616283,1.0,2497.39,209934,32.05 +272,1500,2.665154,1.0,2495.6,210084,32.07 +273,1500,2.688141,1.0,2504.76,209316,31.96 +274,1500,2.728167,1.0,2500.42,209679,32.01 +275,1500,2.761949,1.0,2499.35,209770,32.03 +276,1500,2.759468,1.0,2501.02,209629,32.01 +277,1500,2.73421,1.0,2505.11,209287,31.95 +278,1500,2.710704,1.0,2498.71,209823,32.03 +279,1500,2.695947,1.0,2500.23,209695,32.02 +280,1500,2.661022,1.0,2502.46,209509,31.99 +281,1500,2.661178,1.0,2500.31,209689,32.01 +282,1500,2.656587,1.0,2496.01,210050,32.07 +283,1500,2.667393,1.0,2497.85,209895,32.05 +284,1500,2.645016,1.0,2499.81,209731,32.02 +285,1500,2.658358,1.0,2497.54,209921,32.05 +286,1500,2.648874,1.0,2499.08,209792,32.03 +287,1500,2.643574,1.0,2501.42,209595,32.0 +288,1500,2.632357,1.0,2498.17,209868,32.04 +289,1500,2.605199,1.0,2500.52,209671,32.01 +290,1500,2.599687,1.0,2500.76,209651,32.01 +291,1500,2.565944,1.0,2498.25,209862,32.04 +292,1500,2.578022,1.0,2498.69,209825,32.04 +293,1500,2.594464,1.0,2498.45,209845,32.04 +294,1500,2.617919,1.0,2497.29,209943,32.05 +295,1500,2.630398,1.0,2500.08,209708,32.02 +296,1500,2.588017,1.0,2498.17,209868,32.04 +297,1500,2.613355,1.0,2498.17,209868,32.04 +298,1500,2.601538,1.0,2764.71,189635,28.95 +299,1500,2.58251,1.0,2498.32,209855,32.04 +300,1500,2.581956,1.0,2493.35,210274,32.1 +301,1500,2.57077,1.0,2504.38,209348,31.96 +302,1500,2.582061,1.0,2504.64,209326,31.96 +303,1500,2.599564,1.0,2496.45,210013,32.06 +304,1500,2.617667,1.0,2502.67,209491,31.98 +305,1500,2.607164,1.0,2499.88,209725,32.02 +306,1500,2.613627,1.0,2509.25,208942,31.9 +307,1500,2.58402,1.0,2498.86,209810,32.03 +308,1500,2.639379,1.0,2504.32,209353,31.96 +309,1500,2.666293,1.0,2499.62,209746,32.02 +310,1500,2.689088,1.0,2504.0,209379,31.97 +311,1500,2.67249,1.0,2504.04,209377,31.97 +312,1500,2.624428,1.0,2499.85,209727,32.02 +313,1500,2.686193,1.0,2501.23,209612,32.0 +314,1500,2.658987,1.0,2503.55,209418,31.97 +315,1500,2.668251,1.0,2496.38,210019,32.06 +316,1500,2.661666,1.0,2503.78,209398,31.97 +317,1500,2.642533,1.0,2502.36,209517,31.99 +318,1500,2.658486,1.0,2501.67,209575,32.0 +319,1500,2.665368,1.0,2501.86,209558,31.99 +320,1500,2.652778,1.0,2503.0,209463,31.98 +321,1500,2.658043,1.0,2498.38,209851,32.04 +322,1500,2.681621,1.0,2501.26,209609,32.0 +323,1500,2.668596,1.0,2502.51,209504,31.99 +324,1500,2.63235,1.0,2501.0,209631,32.01 +325,1500,2.624731,1.0,2504.15,209367,31.97 +326,1500,2.591722,1.0,2503.18,209448,31.98 +327,1500,2.547419,1.0,2505.53,209252,31.95 +328,1500,2.578428,1.0,2498.67,209827,32.04 +329,1500,2.560731,1.0,2500.75,209652,32.01 +330,1500,2.585317,1.0,2497.66,209912,32.05 +331,1500,2.612563,1.0,2501.03,209628,32.0 +332,1500,2.6228,1.0,2499.4,209765,32.03 +333,1500,2.609684,1.0,2503.44,209426,31.97 +334,1500,2.639728,1.0,2501.09,209624,32.0 +335,1500,2.655841,1.0,2496.66,209995,32.06 +336,1500,2.605947,1.0,2501.92,209554,31.99 +337,1500,2.62074,1.0,2503.39,209431,31.97 +338,1500,2.611902,1.0,2496.71,209991,32.06 +339,1500,2.629633,1.0,2504.56,209333,31.96 +340,1500,2.595688,1.0,2498.92,209805,32.03 +341,1500,2.599709,1.0,2501.73,209570,32.0 +342,1500,2.642176,1.0,2498.07,209877,32.04 +343,1500,2.61869,1.0,2497.95,209886,32.04 +344,1500,2.640408,1.0,2500.23,209696,32.02 +345,1500,2.651068,1.0,2499.25,209777,32.03 +346,1500,2.640402,1.0,2496.41,210016,32.06 +347,1500,2.640979,1.0,2496.47,210011,32.06 +348,1500,2.625766,1.0,2505.88,209223,31.94 +349,1500,2.638305,1.0,2504.49,209339,31.96 +350,1500,2.63173,1.0,2502.65,209492,31.98 +351,1500,2.678804,1.0,2499.11,209789,32.03 +352,1500,2.66285,1.0,2499.22,209780,32.03 +353,1500,2.642324,1.0,2500.14,209703,32.02 +354,1500,2.68531,1.0,2504.34,209351,31.96 +355,1500,2.695756,1.0,2500.28,209692,32.01 +356,1500,2.614161,1.0,2501.99,209548,31.99 +357,1500,2.627071,1.0,2500.09,209707,32.02 +358,1500,2.638768,1.0,2497.49,209926,32.05 +359,1500,2.60865,1.0,2499.26,209777,32.03 +360,1500,2.622549,1.0,2499.18,209783,32.03 +361,1500,2.607933,1.0,2496.72,209991,32.06 +362,1500,2.607696,1.0,2496.08,210044,32.07 +363,1500,2.595557,1.0,2498.71,209823,32.03 +364,1500,2.650946,1.0,2498.88,209809,32.03 +365,1500,2.641341,1.0,2502.68,209490,31.98 +366,1500,2.646917,1.0,2502.62,209495,31.98 +367,1500,2.635313,1.0,2496.63,209998,32.06 +368,1500,2.604769,1.0,2498.86,209811,32.03 +369,1500,2.614996,1.0,2497.13,209956,32.06 +370,1500,2.61447,1.0,2499.68,209742,32.02 +371,1500,2.626087,1.0,2503.0,209463,31.98 +372,1500,2.616861,1.0,2502.16,209534,31.99 +373,1500,2.58394,1.0,2499.57,209751,32.02 +374,1500,2.53583,1.0,2499.88,209725,32.02 +375,1500,2.537096,1.0,2502.59,209498,31.99 +376,1500,2.543708,1.0,2497.4,209933,32.05 +377,1500,2.54069,1.0,2499.48,209758,32.02 +378,1500,2.534647,1.0,2500.09,209707,32.02 +379,1500,2.458322,1.0,3008.53,174266,26.61 +380,1500,2.478253,1.0,2498.42,209847,32.04 +381,1500,2.439136,1.0,2497.76,209903,32.05 +382,1500,2.367838,1.0,2499.43,209762,32.03 +383,1500,2.388532,1.0,2498.49,209842,32.04 +384,1500,2.43778,1.0,2500.15,209702,32.02 +385,1500,2.414677,1.0,2497.87,209893,32.05 +386,1500,2.424039,1.0,2502.17,209533,31.99 +387,1500,2.392342,1.0,2495.37,210104,32.08 +388,1500,2.331812,1.0,2497.23,209947,32.05 +389,1500,2.366487,1.0,2502.65,209493,31.98 +390,1500,2.32739,1.0,2497.37,209935,32.05 +391,1500,2.420736,1.0,2498.26,209861,32.04 +392,1500,2.435264,1.0,2501.12,209620,32.0 +393,1500,2.430283,1.0,2502.73,209486,31.98 +394,1500,2.462935,1.0,2497.48,209927,32.05 +395,1500,2.477935,1.0,2504.88,209306,31.96 +396,1500,2.477937,1.0,2499.35,209769,32.03 +397,1500,2.507707,1.0,2505.31,209270,31.95 +398,1500,2.508839,1.0,2501.14,209619,32.0 +399,1500,2.461043,1.0,2498.06,209877,32.04 +400,1500,2.516207,1.0,2497.98,209885,32.04 +401,1500,2.5199,1.0,2504.08,209373,31.97 +402,1500,2.501173,1.0,2502.29,209523,31.99 +403,1500,2.533427,1.0,2501.77,209566,32.0 +404,1500,2.571604,1.0,2503.99,209381,31.97 +405,1500,2.573077,1.0,2500.14,209703,32.02 +406,1500,2.582443,1.0,2504.71,209321,31.96 +407,1500,2.595983,1.0,2499.64,209745,32.02 +408,1500,2.582707,1.0,2502.59,209498,31.99 +409,1500,2.59155,1.0,2497.45,209929,32.05 +410,1500,2.597649,1.0,2500.9,209640,32.01 +411,1500,2.615967,1.0,2501.65,209576,32.0 +412,1500,2.592289,1.0,2498.68,209826,32.04 +413,1500,2.633285,1.0,2503.42,209428,31.97 +414,1500,2.634857,1.0,2502.98,209465,31.98 +415,1500,2.646015,1.0,2496.7,209992,32.06 +416,1500,2.659769,1.0,2498.94,209804,32.03 +417,1500,2.644014,1.0,2504.6,209329,31.96 +418,1500,2.642324,1.0,2503.14,209452,31.98 +419,1500,2.610305,1.0,2503.3,209438,31.98 +420,1500,2.601027,1.0,2500.63,209662,32.01 +421,1500,2.611735,1.0,2502.53,209503,31.99 +422,1500,2.596965,1.0,2500.54,209670,32.01 +423,1500,2.610565,1.0,2498.29,209858,32.04 +424,1500,2.614224,1.0,2498.66,209828,32.04 +425,1500,2.633454,1.0,2500.31,209689,32.01 +426,1500,2.653873,1.0,2495.96,210054,32.07 +427,1500,2.624991,1.0,2498.74,209821,32.03 +428,1500,2.600636,1.0,2504.57,209332,31.96 +429,1500,2.589331,1.0,2496.92,209973,32.06 +430,1500,2.603567,1.0,2502.68,209490,31.98 +431,1500,2.654011,1.0,2498.71,209823,32.03 +432,1500,2.55225,1.0,2499.25,209777,32.03 +433,1500,2.576312,1.0,2501.86,209559,31.99 +434,1500,2.587704,1.0,2500.07,209709,32.02 +435,1500,2.696486,1.0,2502.16,209534,31.99 +436,1500,2.713219,1.0,2498.66,209827,32.04 +437,1500,2.742426,1.0,2500.29,209691,32.01 +438,1500,2.716522,1.0,2502.57,209499,31.99 +439,1500,2.714635,1.0,2499.53,209754,32.02 +440,1500,2.735734,1.0,2503.85,209392,31.97 +441,1500,2.716592,1.0,2502.81,209479,31.98 +442,1500,2.679167,1.0,2504.39,209347,31.96 +443,1500,2.652363,1.0,2503.08,209456,31.98 +444,1500,2.668071,1.0,2496.79,209984,32.06 +445,1500,2.665459,1.0,2502.05,209543,31.99 +446,1500,2.679841,1.0,2501.16,209617,32.0 +447,1500,2.669771,1.0,2502.88,209473,31.98 +448,1500,2.672466,1.0,2500.79,209648,32.01 +449,1500,2.641235,1.0,2503.18,209448,31.98 +450,1500,2.633718,1.0,2502.95,209467,31.98 +451,1500,2.610007,1.0,2504.55,209334,31.96 +452,1500,2.55863,1.0,2494.24,210199,32.09 +453,1500,2.53251,1.0,2500.48,209674,32.01 +454,1500,2.527563,1.0,2501.22,209612,32.0 +455,1500,2.534919,1.0,2504.73,209319,31.96 +456,1500,2.523789,1.0,2499.57,209750,32.02 +457,1500,2.497783,1.0,2501.66,209575,32.0 +458,1500,2.515548,1.0,2502.16,209534,31.99 +459,1500,2.558479,1.0,2500.55,209668,32.01 +460,1500,2.568665,1.0,2601.98,201495,30.76 +461,1500,2.549469,1.0,2497.65,209912,32.05 +462,1500,2.527152,1.0,2497.69,209908,32.05 +463,1500,2.562973,1.0,2499.69,209741,32.02 +464,1500,2.558161,1.0,2496.52,210007,32.06 +465,1500,2.654085,1.0,2503.38,209432,31.98 +466,1500,2.631398,1.0,2498.73,209821,32.03 +467,1500,2.654613,1.0,2506.62,209161,31.93 +468,1500,2.650343,1.0,2500.46,209676,32.01 +469,1500,2.615141,1.0,2498.96,209802,32.03 +470,1500,2.604841,1.0,2500.3,209689,32.01 +471,1500,2.579015,1.0,2496.6,210001,32.06 +472,1500,2.538458,1.0,2496.98,209968,32.06 +473,1500,2.529463,1.0,2498.7,209824,32.03 +474,1500,2.578395,1.0,2499.59,209749,32.02 +475,1500,2.591929,1.0,2501.67,209575,32.0 +476,1500,2.588667,1.0,2504.15,209367,31.97 +477,1500,2.54918,1.0,2498.01,209882,32.04 +478,1500,2.571854,1.0,2501.14,209619,32.0 +479,1500,2.581863,1.0,2495.07,210129,32.08 +480,1500,2.540622,1.0,2502.05,209543,31.99 +481,1500,2.452574,1.0,2504.77,209315,31.96 +482,1500,2.525343,1.0,2500.68,209658,32.01 +483,1500,2.55024,1.0,2496.04,210048,32.07 +484,1500,2.497811,1.0,2498.28,209859,32.04 +485,1500,2.496545,1.0,2499.34,209770,32.03 +486,1500,2.525746,1.0,2503.49,209423,31.97 +487,1500,2.539668,1.0,2497.55,209921,32.05 +488,1500,2.530516,1.0,2500.24,209694,32.02 +489,1500,2.537329,1.0,2501.13,209620,32.0 +490,1500,2.534464,1.0,2503.14,209452,31.98 +491,1500,2.530831,1.0,2503.65,209409,31.97 +492,1500,2.498196,1.0,2500.75,209652,32.01 +493,1500,2.483597,1.0,2497.92,209889,32.04 +494,1500,2.509084,1.0,2502.0,209547,31.99 +495,1500,2.470687,1.0,2506.07,209207,31.94 +496,1500,2.47986,1.0,2502.48,209507,31.99 +497,1500,2.52902,1.0,2496.55,210005,32.06 +498,1500,2.506069,1.0,2496.61,209999,32.06 +499,1500,2.503669,1.0,2502.22,209528,31.99 +500,1500,2.500904,1.0,2475.99,211748,32.33 +501,1500,2.463526,1.0,2481.54,211275,32.26 +502,1500,2.518413,1.0,2484.65,211011,32.22 +503,1500,2.577312,1.0,2492.33,210360,32.12 +504,1500,2.613699,1.0,2491.09,210465,32.13 +505,1500,2.619416,1.0,2496.58,210002,32.06 +506,1500,2.636152,1.0,2498.08,209876,32.04 +507,1500,2.624096,1.0,2502.37,209516,31.99 +508,1500,2.641861,1.0,2499.42,209763,32.03 +509,1500,2.601222,1.0,2505.3,209271,31.95 +510,1500,2.568798,1.0,2501.78,209565,32.0 +511,1500,2.529884,1.0,2493.74,210241,32.1 +512,1500,2.572685,1.0,2497.54,209922,32.05 +513,1500,2.588677,1.0,2498.74,209821,32.03 +514,1500,2.607854,1.0,2497.14,209955,32.05 +515,1500,2.605475,1.0,2506.18,209198,31.94 +516,1500,2.625303,1.0,2497.45,209929,32.05 +517,1500,2.591898,1.0,2498.23,209864,32.04 +518,1500,2.607237,1.0,2509.66,208907,31.89 +519,1500,2.591363,1.0,2502.36,209517,31.99 +520,1500,2.605786,1.0,2501.47,209592,32.0 +521,1500,2.601866,1.0,2498.37,209851,32.04 +522,1500,2.550722,1.0,2501.55,209585,32.0 +523,1500,2.519845,1.0,2502.27,209524,31.99 +524,1500,2.54552,1.0,2502.08,209540,31.99 +525,1500,2.555268,1.0,2502.19,209531,31.99 +526,1500,2.545308,1.0,2495.97,210053,32.07 +527,1500,2.570636,1.0,2502.35,209518,31.99 +528,1500,2.579141,1.0,2495.69,210077,32.07 +529,1500,2.580171,1.0,2498.88,209808,32.03 +530,1500,2.548039,1.0,2500.94,209636,32.01 +531,1500,2.50298,1.0,2500.77,209650,32.01 +532,1500,2.525493,1.0,2501.45,209594,32.0 +533,1500,2.506324,1.0,2499.94,209720,32.02 +534,1500,2.501816,1.0,2501.13,209620,32.0 +535,1500,2.491864,1.0,2504.63,209327,31.96 +536,1500,2.48644,1.0,2500.08,209708,32.02 +537,1500,2.434391,1.0,2500.46,209676,32.01 +538,1500,2.410299,1.0,2500.8,209648,32.01 +539,1500,2.355827,1.0,2500.74,209652,32.01 +540,1500,2.366697,1.0,2501.53,209586,32.0 +541,1500,2.375723,1.0,3375.97,155299,23.71 +542,1500,2.350408,1.0,2494.44,210182,32.09 +543,1500,2.399915,1.0,2495.23,210116,32.08 +544,1500,2.404802,1.0,2497.91,209890,32.05 +545,1500,2.411226,1.0,2502.94,209469,31.98 +546,1500,2.450549,1.0,2499.5,209757,32.02 +547,1500,2.512951,1.0,2501.35,209601,32.0 +548,1500,2.452717,1.0,2497.34,209938,32.05 +549,1500,2.430202,1.0,2501.5,209589,32.0 +550,1500,2.483732,1.0,2498.78,209817,32.03 +551,1500,2.436023,1.0,2502.61,209496,31.98 +552,1500,2.37089,1.0,2501.94,209552,31.99 +553,1500,2.39808,1.0,2499.48,209759,32.02 +554,1500,2.441161,1.0,2499.22,209780,32.03 +555,1500,2.44297,1.0,2515.28,208440,31.82 +556,1500,2.426135,1.0,2497.45,209928,32.05 +557,1500,2.446479,1.0,2499.78,209733,32.02 +558,1500,2.498584,1.0,2495.45,210097,32.08 +559,1500,2.485985,1.0,2502.52,209503,31.99 +560,1500,2.494606,1.0,2500.85,209644,32.01 +561,1500,2.515181,1.0,2504.71,209321,31.96 +562,1500,2.476007,1.0,2500.38,209682,32.01 +563,1500,2.465159,1.0,2500.36,209685,32.01 +564,1500,2.490591,1.0,2500.24,209695,32.02 +565,1500,2.511817,1.0,2498.87,209810,32.03 +566,1500,2.482246,1.0,2497.28,209943,32.05 +567,1500,2.484301,1.0,2500.44,209678,32.01 +568,1500,2.439863,1.0,2499.93,209720,32.02 +569,1500,2.414052,1.0,2504.69,209322,31.96 +570,1500,2.429948,1.0,2497.33,209939,32.05 +571,1500,2.440057,1.0,2500.07,209709,32.02 +572,1500,2.492334,1.0,2498.79,209817,32.03 +573,1500,2.540238,1.0,2500.43,209679,32.01 +574,1500,2.579401,1.0,2499.69,209740,32.02 +575,1500,2.539988,1.0,2497.77,209902,32.05 +576,1500,2.479109,1.0,2501.94,209552,31.99 +577,1500,2.549384,1.0,2506.03,209210,31.94 +578,1500,2.503151,1.0,2496.43,210015,32.06 +579,1500,2.48311,1.0,2503.21,209446,31.98 +580,1500,2.403492,1.0,2499.16,209785,32.03 +581,1500,2.474327,1.0,2498.07,209877,32.04 +582,1500,2.492671,1.0,2500.23,209696,32.02 +583,1500,2.491891,1.0,2502.13,209536,31.99 +584,1500,2.457806,1.0,2497.86,209894,32.05 +585,1500,2.450195,1.0,2506.96,209132,31.93 +586,1500,2.478761,1.0,2497.61,209915,32.05 +587,1500,2.517311,1.0,2500.22,209696,32.02 +588,1500,2.535973,1.0,2503.2,209446,31.98 +589,1500,2.500935,1.0,2501.28,209608,32.0 +590,1500,2.467282,1.0,2497.19,209951,32.05 +591,1500,2.462404,1.0,2495.94,210056,32.07 +592,1500,2.442026,1.0,2499.57,209751,32.02 +593,1500,2.445343,1.0,2499.51,209756,32.02 +594,1500,2.444952,1.0,2501.17,209616,32.0 +595,1500,2.50391,1.0,2505.3,209271,31.95 +596,1500,2.517616,1.0,2495.79,210069,32.07 +597,1500,2.523563,1.0,2504.03,209377,31.97 +598,1500,2.520262,1.0,2498.34,209854,32.04 +599,1500,2.546318,1.0,2503.6,209413,31.97 +600,1500,2.573231,1.0,2495.06,210130,32.08 +601,1500,2.590959,1.0,2499.26,209777,32.03 +602,1500,2.623053,1.0,2497.92,209890,32.04 +603,1500,2.657384,1.0,2497.67,209911,32.05 +604,1500,2.634753,1.0,2502.15,209534,31.99 +605,1500,2.635326,1.0,2498.93,209804,32.03 +606,1500,2.613039,1.0,2503.49,209422,31.97 +607,1500,2.566059,1.0,2497.07,209961,32.06 +608,1500,2.564715,1.0,2498.68,209825,32.04 +609,1500,2.534109,1.0,2499.28,209775,32.03 +610,1500,2.484601,1.0,2500.25,209694,32.01 +611,1500,2.481609,1.0,2499.37,209767,32.03 +612,1500,2.523672,1.0,2498.71,209823,32.03 +613,1500,2.490175,1.0,2502.44,209510,31.99 +614,1500,2.499527,1.0,2502.15,209534,31.99 +615,1500,2.466889,1.0,2503.91,209387,31.97 +616,1500,2.544195,1.0,2503.06,209459,31.98 +617,1500,2.536221,1.0,2499.84,209728,32.02 +618,1500,2.591457,1.0,2498.25,209861,32.04 +619,1500,2.605541,1.0,2502.96,209467,31.98 +620,1500,2.558934,1.0,2501.61,209580,32.0 +621,1500,2.563063,1.0,2645.23,198201,30.26 +622,1500,2.549821,1.0,2501.61,209580,32.0 +623,1500,2.532256,1.0,2500.5,209673,32.01 +624,1500,2.536579,1.0,2506.06,209207,31.94 +625,1500,2.58213,1.0,2497.51,209924,32.05 +626,1500,2.585435,1.0,2497.79,209900,32.05 +627,1500,2.578101,1.0,2499.72,209739,32.02 +628,1500,2.586965,1.0,2499.31,209773,32.03 +629,1500,2.558031,1.0,2503.92,209386,31.97 +630,1500,2.555793,1.0,2499.98,209717,32.02 +631,1500,2.562231,1.0,2503.98,209382,31.97 +632,1500,2.553953,1.0,2498.23,209863,32.04 +633,1500,2.562541,1.0,2496.87,209978,32.06 +634,1500,2.567383,1.0,2499.62,209747,32.02 +635,1500,2.529419,1.0,2504.54,209334,31.96 +636,1500,2.557127,1.0,2500.53,209670,32.01 +637,1500,2.55562,1.0,2499.21,209781,32.03 +638,1500,2.502341,1.0,2498.16,209869,32.04 +639,1500,2.480235,1.0,2497.12,209957,32.06 +640,1500,2.470096,1.0,2499.12,209788,32.03 +641,1500,2.46014,1.0,2494.71,210159,32.09 +642,1500,2.454784,1.0,2506.16,209199,31.94 +643,1500,2.453636,1.0,2504.26,209358,31.96 +644,1500,2.493934,1.0,2498.31,209857,32.04 +645,1500,2.482507,1.0,2497.23,209948,32.05 +646,1500,2.500603,1.0,2501.29,209607,32.0 +647,1500,2.483368,1.0,2503.37,209432,31.98 +648,1500,2.496198,1.0,2500.98,209632,32.01 +649,1500,2.470514,1.0,2499.53,209754,32.02 +650,1500,2.483934,1.0,2501.27,209608,32.0 +651,1500,2.463478,1.0,2505.83,209227,31.94 +652,1500,2.462173,1.0,2504.54,209335,31.96 +653,1500,2.452519,1.0,2499.08,209792,32.03 +654,1500,2.466888,1.0,2501.52,209587,32.0 +655,1500,2.389489,1.0,2502.1,209539,31.99 +656,1500,2.398192,1.0,2500.36,209685,32.01 +657,1500,2.4248,1.0,2498.61,209832,32.04 +658,1500,2.439903,1.0,2501.29,209606,32.0 +659,1500,2.458521,1.0,2500.78,209649,32.01 +660,1500,2.462703,1.0,2509.84,208892,31.89 +661,1500,2.472215,1.0,2502.8,209480,31.98 +662,1500,2.471677,1.0,2504.38,209348,31.96 +663,1500,2.462579,1.0,2502.03,209544,31.99 +664,1500,2.475935,1.0,2501.32,209604,32.0 +665,1500,2.480156,1.0,2495.91,210058,32.07 +666,1500,2.508779,1.0,2496.96,209970,32.06 +667,1500,2.47159,1.0,2506.19,209197,31.94 +668,1500,2.491536,1.0,2499.73,209738,32.02 +669,1500,2.513735,1.0,2501.13,209620,32.0 +670,1500,2.491293,1.0,2498.09,209875,32.04 +671,1500,2.493976,1.0,2495.46,210096,32.08 +672,1500,2.483517,1.0,2496.53,210006,32.06 +673,1500,2.500287,1.0,2499.84,209728,32.02 +674,1500,2.473263,1.0,2500.21,209697,32.02 +675,1500,2.487736,1.0,2500.47,209675,32.01 +676,1500,2.477532,1.0,2504.8,209313,31.96 +677,1500,2.454688,1.0,2499.5,209757,32.02 +678,1500,2.430004,1.0,2498.98,209800,32.03 +679,1500,2.466982,1.0,2505.25,209276,31.95 +680,1500,2.479966,1.0,2495.83,210066,32.07 +681,1500,2.485721,1.0,2498.94,209804,32.03 +682,1500,2.496,1.0,2501.68,209574,32.0 +683,1500,2.475892,1.0,2499.28,209775,32.03 +684,1500,2.46171,1.0,2503.92,209386,31.97 +685,1500,2.436354,1.0,2495.77,210070,32.07 +686,1500,2.415804,1.0,2501.18,209616,32.0 +687,1500,2.399857,1.0,2498.45,209845,32.04 +688,1500,2.421625,1.0,2502.31,209521,31.99 +689,1500,2.439345,1.0,2501.34,209602,32.0 +690,1500,2.447331,1.0,2498.95,209803,32.03 +691,1500,2.425838,1.0,2500.67,209658,32.01 +692,1500,2.469981,1.0,2496.11,210041,32.07 +693,1500,2.510274,1.0,2501.9,209555,31.99 +694,1500,2.501991,1.0,2503.5,209422,31.97 +695,1500,2.533423,1.0,2502.78,209482,31.98 +696,1500,2.5161,1.0,2501.67,209575,32.0 +697,1500,2.49033,1.0,2502.44,209510,31.99 +698,1500,2.467871,1.0,2501.35,209601,32.0 +699,1500,2.510379,1.0,2501.73,209570,32.0 +700,1500,2.376646,1.0,2498.71,209823,32.03 +701,1500,2.384216,1.0,2501.5,209589,32.0 +702,1500,2.357306,1.0,2856.88,183517,28.02 +703,1500,2.364936,1.0,2500.41,209680,32.01 +704,1500,2.426741,1.0,2499.82,209730,32.02 +705,1500,2.42927,1.0,2499.99,209716,32.02 +706,1500,2.431152,1.0,2499.87,209726,32.02 +707,1500,2.440887,1.0,2501.34,209603,32.0 +708,1500,2.427637,1.0,2502.4,209514,31.99 +709,1500,2.426681,1.0,2507.04,209125,31.93 +710,1500,2.447408,1.0,2499.23,209780,32.03 +711,1500,2.485365,1.0,2502.19,209531,31.99 +712,1500,2.525821,1.0,2506.1,209204,31.94 +713,1500,2.563213,1.0,2503.93,209385,31.97 +714,1500,2.450297,1.0,2497.82,209898,32.05 +715,1500,2.448795,1.0,2497.33,209939,32.05 +716,1500,2.484999,1.0,2501.28,209607,32.0 +717,1500,2.458955,1.0,2502.92,209470,31.98 +718,1500,2.459789,1.0,2496.35,210021,32.06 +719,1500,2.461536,1.0,2501.18,209615,32.0 +720,1500,2.508784,1.0,2502.23,209527,31.99 +721,1500,2.546022,1.0,2499.04,209795,32.03 +722,1500,2.569562,1.0,2496.47,210011,32.06 +723,1500,2.549892,1.0,2498.75,209820,32.03 +724,1500,2.553147,1.0,2497.75,209904,32.05 +725,1500,2.513919,1.0,2502.86,209475,31.98 +726,1500,2.511291,1.0,2501.91,209555,31.99 +727,1500,2.526622,1.0,2502.54,209502,31.99 +728,1500,2.583192,1.0,2501.89,209556,31.99 +729,1500,2.570209,1.0,2502.26,209525,31.99 +730,1500,2.555771,1.0,2501.04,209627,32.0 +731,1500,2.594396,1.0,2498.95,209803,32.03 +732,1500,2.598548,1.0,2499.45,209761,32.03 +733,1500,2.575133,1.0,2503.75,209401,31.97 +734,1500,2.570469,1.0,2498.43,209847,32.04 +735,1500,2.577838,1.0,2497.93,209889,32.04 +736,1500,2.562257,1.0,2501.84,209560,31.99 +737,1500,2.51228,1.0,2497.68,209910,32.05 +738,1500,2.525939,1.0,2500.16,209701,32.02 +739,1500,2.48764,1.0,2500.39,209682,32.01 +740,1500,2.484023,1.0,2503.1,209455,31.98 +741,1500,2.495674,1.0,2505.57,209249,31.95 +742,1500,2.493286,1.0,2500.44,209678,32.01 +743,1500,2.512838,1.0,2506.23,209193,31.94 +744,1500,2.526144,1.0,2499.91,209722,32.02 +745,1500,2.485605,1.0,2502.44,209511,31.99 +746,1500,2.502808,1.0,2496.03,210048,32.07 +747,1500,2.541489,1.0,2500.48,209674,32.01 +748,1500,2.580649,1.0,2500.65,209660,32.01 +749,1500,2.583109,1.0,2500.45,209677,32.01 +750,1500,2.613416,1.0,2475.46,211794,32.34 +751,1500,2.575135,1.0,2483.76,211086,32.23 +752,1500,2.561207,1.0,2488.51,210683,32.17 +753,1500,2.551033,1.0,2489.22,210623,32.16 +754,1500,2.522961,1.0,2494.46,210181,32.09 +755,1500,2.514381,1.0,2496.97,209969,32.06 +756,1500,2.539011,1.0,2498.62,209831,32.04 +757,1500,2.512881,1.0,2496.05,210046,32.07 +758,1500,2.500824,1.0,2506.59,209164,31.93 +759,1500,2.456874,1.0,2510.1,208870,31.89 +760,1500,2.449276,1.0,2505.61,209245,31.95 +761,1500,2.460873,1.0,2511.35,208767,31.87 +762,1500,2.46321,1.0,2503.8,209397,31.97 +763,1500,2.438271,1.0,2504.37,209349,31.96 +764,1500,2.398702,1.0,2511.76,208733,31.87 +765,1500,2.37854,1.0,2504.58,209331,31.96 +766,1500,2.396469,1.0,2506.18,209198,31.94 +767,1500,2.431147,1.0,2501.81,209563,32.0 +768,1500,2.426193,1.0,2500.47,209675,32.01 +769,1500,2.418622,1.0,2503.9,209388,31.97 +770,1500,2.455873,1.0,2497.98,209884,32.04 +771,1500,2.489749,1.0,2500.25,209694,32.01 +772,1500,2.477514,1.0,2502.03,209544,31.99 +773,1500,2.490943,1.0,2499.93,209721,32.02 +774,1500,2.454402,1.0,2500.35,209685,32.01 +775,1500,2.453161,1.0,2500.67,209659,32.01 +776,1500,2.457556,1.0,2502.05,209543,31.99 +777,1500,2.470391,1.0,2496.88,209976,32.06 +778,1500,2.43666,1.0,2500.8,209648,32.01 +779,1500,2.393297,1.0,2497.66,209912,32.05 +780,1500,2.393302,1.0,2496.59,210001,32.06 +781,1500,2.438059,1.0,2499.5,209757,32.02 +782,1500,2.459424,1.0,2503.23,209444,31.98 +783,1500,2.507362,1.0,2499.13,209788,32.03 +784,1500,2.479795,1.0,2796.38,187487,28.62 +785,1500,2.443444,1.0,2500.97,209634,32.01 +786,1500,2.376552,1.0,2499.26,209777,32.03 +787,1500,2.346382,1.0,2496.73,209989,32.06 +788,1500,2.339653,1.0,2501.14,209619,32.0 +789,1500,2.38525,1.0,2503.26,209441,31.98 +790,1500,2.442078,1.0,2499.15,209786,32.03 +791,1500,2.409874,1.0,2499.38,209767,32.03 +792,1500,2.37451,1.0,2498.75,209820,32.03 +793,1500,2.398126,1.0,2498.92,209805,32.03 +794,1500,2.397676,1.0,2499.77,209734,32.02 +795,1500,2.424432,1.0,2500.29,209690,32.01 +796,1500,2.421531,1.0,2500.98,209633,32.01 +797,1500,2.430554,1.0,2500.53,209670,32.01 +798,1500,2.479384,1.0,2496.81,209982,32.06 +799,1500,2.448715,1.0,2501.1,209623,32.0 +800,1500,2.464734,1.0,2497.15,209954,32.05 +801,1500,2.485478,1.0,2497.07,209961,32.06 +802,1500,2.503597,1.0,2512.89,208639,31.85 +803,1500,2.497169,1.0,2495.1,210126,32.08 +804,1500,2.495256,1.0,2498.36,209853,32.04 +805,1500,2.459168,1.0,2496.95,209971,32.06 +806,1500,2.542786,1.0,2499.63,209746,32.02 +807,1500,2.502023,1.0,2498.68,209826,32.04 +808,1500,2.530503,1.0,2498.87,209810,32.03 +809,1500,2.506526,1.0,2499.72,209738,32.02 +810,1500,2.499926,1.0,2499.17,209784,32.03 +811,1500,2.51162,1.0,2499.7,209740,32.02 +812,1500,2.518845,1.0,2500.26,209693,32.01 +813,1500,2.486244,1.0,2497.5,209925,32.05 +814,1500,2.483264,1.0,2502.02,209545,31.99 +815,1500,2.496015,1.0,2500.6,209665,32.01 +816,1500,2.470131,1.0,2501.89,209556,31.99 +817,1500,2.521402,1.0,2499.69,209741,32.02 +818,1500,2.530285,1.0,2497.64,209913,32.05 +819,1500,2.537301,1.0,2504.44,209343,31.96 +820,1500,2.521476,1.0,2498.51,209840,32.04 +821,1500,2.536437,1.0,2500.13,209704,32.02 +822,1500,2.523154,1.0,2501.26,209609,32.0 +823,1500,2.503628,1.0,2505.43,209260,31.95 +824,1500,2.489208,1.0,2497.59,209917,32.05 +825,1500,2.499214,1.0,2499.67,209742,32.02 +826,1500,2.467831,1.0,2499.25,209777,32.03 +827,1500,2.434501,1.0,2498.28,209859,32.04 +828,1500,2.413911,1.0,2500.52,209671,32.01 +829,1500,2.417626,1.0,2497.0,209966,32.06 +830,1500,2.41285,1.0,2500.08,209708,32.02 +831,1500,2.401505,1.0,2501.14,209619,32.0 +832,1500,2.387107,1.0,2500.33,209687,32.01 +833,1500,2.388825,1.0,2503.41,209429,31.97 +834,1500,2.321134,1.0,2502.12,209537,31.99 +835,1500,2.310482,1.0,2498.86,209811,32.03 +836,1500,2.296415,1.0,2498.59,209833,32.04 +837,1500,2.241724,1.0,2497.2,209950,32.05 +838,1500,2.274436,1.0,2497.66,209911,32.05 +839,1500,2.280051,1.0,2498.69,209825,32.03 +840,1500,2.259181,1.0,2501.94,209552,31.99 +841,1500,2.31905,1.0,2498.99,209799,32.03 +842,1500,2.335775,1.0,2498.74,209820,32.03 +843,1500,2.365811,1.0,2500.41,209680,32.01 +844,1500,2.361053,1.0,2501.03,209628,32.0 +845,1500,2.376585,1.0,2505.31,209270,31.95 +846,1500,2.384495,1.0,2502.76,209483,31.98 +847,1500,2.378119,1.0,2505.06,209291,31.95 +848,1500,2.429506,1.0,2501.69,209573,32.0 +849,1500,2.447713,1.0,2503.76,209400,31.97 +850,1500,2.429006,1.0,2502.71,209488,31.98 +851,1500,2.430386,1.0,2501.07,209625,32.0 +852,1500,2.461675,1.0,2500.5,209672,32.01 +853,1500,2.427366,1.0,2497.99,209883,32.04 +854,1500,2.420453,1.0,2500.12,209705,32.02 +855,1500,2.422431,1.0,2506.84,209142,31.93 +856,1500,2.393103,1.0,2504.09,209372,31.97 +857,1500,2.413194,1.0,2499.06,209794,32.03 +858,1500,2.432611,1.0,2498.59,209833,32.04 +859,1500,2.446908,1.0,2505.13,209286,31.95 +860,1500,2.430805,1.0,2503.85,209392,31.97 +861,1500,2.454606,1.0,2499.66,209743,32.02 +862,1500,2.432909,1.0,2504.19,209364,31.96 +863,1500,2.46064,1.0,2498.95,209803,32.03 +864,1500,2.466855,1.0,2495.95,210055,32.07 +865,1500,2.488819,1.0,2751.02,190579,29.1 +866,1500,2.46092,1.0,2502.21,209529,31.99 +867,1500,2.434904,1.0,2503.9,209388,31.97 +868,1500,2.458098,1.0,2513.55,208584,31.85 +869,1500,2.449237,1.0,2497.8,209900,32.05 +870,1500,2.4485,1.0,2497.95,209887,32.04 +871,1500,2.446891,1.0,2500.76,209651,32.01 +872,1500,2.394001,1.0,2496.83,209981,32.06 +873,1500,2.374766,1.0,2502.69,209489,31.98 +874,1500,2.393871,1.0,2499.73,209737,32.02 +875,1500,2.388008,1.0,2502.98,209465,31.98 +876,1500,2.403612,1.0,2499.08,209792,32.03 +877,1500,2.423884,1.0,2497.25,209945,32.05 +878,1500,2.443265,1.0,2504.74,209318,31.96 +879,1500,2.426598,1.0,2495.43,210099,32.08 +880,1500,2.472516,1.0,2498.87,209810,32.03 +881,1500,2.499515,1.0,2500.67,209658,32.01 +882,1500,2.513423,1.0,2499.99,209715,32.02 +883,1500,2.552547,1.0,2502.99,209465,31.98 +884,1500,2.552526,1.0,2504.21,209362,31.96 +885,1500,2.534397,1.0,2501.25,209610,32.0 +886,1500,2.541211,1.0,2500.06,209710,32.02 +887,1500,2.519362,1.0,2499.03,209796,32.03 +888,1500,2.499014,1.0,2501.62,209579,32.0 +889,1500,2.477072,1.0,2503.42,209428,31.97 +890,1500,2.486483,1.0,2500.29,209690,32.01 +891,1500,2.499196,1.0,2498.86,209810,32.03 +892,1500,2.511819,1.0,2497.76,209903,32.05 +893,1500,2.506047,1.0,2498.67,209826,32.04 +894,1500,2.506997,1.0,2504.83,209310,31.96 +895,1500,2.437824,1.0,2494.0,210219,32.1 +896,1500,2.491216,1.0,2500.11,209706,32.02 +897,1500,2.459149,1.0,2506.67,209157,31.93 +898,1500,2.434871,1.0,2497.2,209950,32.05 +899,1500,2.421553,1.0,2498.61,209831,32.04 +900,1500,2.437313,1.0,2495.98,210053,32.07 +901,1500,2.452271,1.0,2499.6,209748,32.02 +902,1500,2.442853,1.0,2496.67,209995,32.06 +903,1500,2.442931,1.0,2499.68,209742,32.02 +904,1500,2.466908,1.0,2496.53,210006,32.06 +905,1500,2.434947,1.0,2499.2,209782,32.03 +906,1500,2.48073,1.0,2498.09,209875,32.04 +907,1500,2.504384,1.0,2499.45,209761,32.03 +908,1500,2.495713,1.0,2505.0,209296,31.95 +909,1500,2.462858,1.0,2499.99,209716,32.02 +910,1500,2.43503,1.0,2505.34,209268,31.95 +911,1500,2.447943,1.0,2502.82,209478,31.98 +912,1500,2.45464,1.0,2493.91,210227,32.1 +913,1500,2.445828,1.0,2498.86,209811,32.03 +914,1500,2.422156,1.0,2495.24,210115,32.08 +915,1500,2.371485,1.0,2498.23,209863,32.04 +916,1500,2.345092,1.0,2501.57,209583,32.0 +917,1500,2.322704,1.0,2503.62,209411,31.97 +918,1500,2.368564,1.0,2499.05,209794,32.03 +919,1500,2.370071,1.0,2501.15,209619,32.0 +920,1500,2.356302,1.0,2502.72,209487,31.98 +921,1500,2.315991,1.0,2504.31,209354,31.96 +922,1500,2.319314,1.0,2499.49,209758,32.02 +923,1500,2.356224,1.0,2499.8,209731,32.02 +924,1500,2.359322,1.0,2498.15,209870,32.04 +925,1500,2.411088,1.0,2497.91,209890,32.04 +926,1500,2.339008,1.0,2499.73,209738,32.02 +927,1500,2.375406,1.0,2503.46,209424,31.97 +928,1500,2.395494,1.0,2501.1,209622,32.0 +929,1500,2.427484,1.0,2500.83,209645,32.01 +930,1500,2.435807,1.0,2502.1,209539,31.99 +931,1500,2.42641,1.0,2504.83,209310,31.96 +932,1500,2.424321,1.0,2494.62,210167,32.09 +933,1500,2.443905,1.0,2501.26,209609,32.0 +934,1500,2.477102,1.0,2502.26,209525,31.99 +935,1500,2.472756,1.0,2503.48,209424,31.97 +936,1500,2.477611,1.0,2495.53,210090,32.08 +937,1500,2.486416,1.0,2504.99,209297,31.95 +938,1500,2.504975,1.0,2502.44,209510,31.99 +939,1500,2.523929,1.0,2512.75,208650,31.86 +940,1500,2.517792,1.0,2499.65,209744,32.02 +941,1500,2.534914,1.0,2499.52,209755,32.02 +942,1500,2.592099,1.0,2508.99,208963,31.9 +943,1500,2.642259,1.0,2506.59,209164,31.93 +944,1500,2.654288,1.0,2500.12,209704,32.02 +945,1500,2.667749,1.0,2504.46,209341,31.96 +946,1500,2.651784,1.0,2864.18,183049,27.95 +947,1500,2.667027,1.0,2502.78,209482,31.98 +948,1500,2.62116,1.0,2505.13,209286,31.95 +949,1500,2.607871,1.0,2499.78,209733,32.02 +950,1500,2.592276,1.0,2497.56,209919,32.05 +951,1500,2.583517,1.0,2507.85,209058,31.92 +952,1500,2.591586,1.0,2495.67,210079,32.07 +953,1500,2.588482,1.0,2496.82,209982,32.06 +954,1500,2.538744,1.0,2500.56,209668,32.01 +955,1500,2.597563,1.0,2499.85,209727,32.02 +956,1500,2.552908,1.0,2500.05,209710,32.02 +957,1500,2.587846,1.0,2499.13,209788,32.03 +958,1500,2.517173,1.0,2499.85,209727,32.02 +959,1500,2.483435,1.0,2500.1,209707,32.02 +960,1500,2.511735,1.0,2497.28,209943,32.05 +961,1500,2.485906,1.0,2499.28,209775,32.03 +962,1500,2.495403,1.0,2500.5,209673,32.01 +963,1500,2.427355,1.0,2501.12,209621,32.0 +964,1500,2.373301,1.0,2503.56,209417,31.97 +965,1500,2.403538,1.0,2497.58,209918,32.05 +966,1500,2.396511,1.0,2499.82,209730,32.02 +967,1500,2.421103,1.0,2502.89,209472,31.98 +968,1500,2.371453,1.0,2506.2,209196,31.94 +969,1500,2.408201,1.0,2500.77,209650,32.01 +970,1500,2.475795,1.0,2500.1,209706,32.02 +971,1500,2.476792,1.0,2498.16,209870,32.04 +972,1500,2.493392,1.0,2502.77,209482,31.98 +973,1500,2.530999,1.0,2504.69,209322,31.96 +974,1500,2.523076,1.0,2498.97,209801,32.03 +975,1500,2.548861,1.0,2503.95,209384,31.97 +976,1500,2.541862,1.0,2500.11,209705,32.02 +977,1500,2.530597,1.0,2501.88,209557,31.99 +978,1500,2.51262,1.0,2500.03,209712,32.02 +979,1500,2.513045,1.0,2501.07,209625,32.0 +980,1500,2.520401,1.0,2500.33,209687,32.01 +981,1500,2.507651,1.0,2498.06,209877,32.04 +982,1500,2.498525,1.0,2498.41,209848,32.04 +983,1500,2.495252,1.0,2504.27,209357,31.96 +984,1500,2.509092,1.0,2504.26,209358,31.96 +985,1500,2.470145,1.0,2498.52,209839,32.04 +986,1500,2.498084,1.0,2504.01,209379,31.97 +987,1500,2.484797,1.0,2503.29,209439,31.98 +988,1500,2.476182,1.0,2522.53,207841,31.73 +989,1500,2.454756,1.0,2502.31,209521,31.99 +990,1500,2.453343,1.0,2504.37,209349,31.96 +991,1500,2.431266,1.0,2498.48,209843,32.04 +992,1500,2.458795,1.0,2502.59,209498,31.99 +993,1500,2.454704,1.0,2504.36,209350,31.96 +994,1500,2.408088,1.0,2496.91,209974,32.06 +995,1500,2.413461,1.0,2498.2,209866,32.04 +996,1500,2.423102,1.0,2515.28,208441,31.82 +997,1500,2.42064,1.0,2495.56,210088,32.08 +998,1500,2.447795,1.0,2500.33,209687,32.01 +999,1500,2.445551,1.0,2499.24,209778,32.03 +1000,1500,2.425055,1.0,2472.32,212063,32.38 +1001,1500,2.43734,1.0,2483.08,211144,32.24 +1002,1500,2.402506,1.0,2487.48,210770,32.18 +1003,1500,2.40229,1.0,2491.41,210438,32.13 +1004,1500,2.383145,1.0,2490.94,210478,32.13 +1005,1500,2.388188,1.0,2496.07,210045,32.07 +1006,1500,2.405605,1.0,2495.65,210080,32.07 +1007,1500,2.419697,1.0,2500.89,209640,32.01 +1008,1500,2.427633,1.0,2499.88,209725,32.02 +1009,1500,2.466596,1.0,2500.15,209702,32.02 +1010,1500,2.444322,1.0,2496.52,210007,32.06 +1011,1500,2.402679,1.0,2498.62,209831,32.04 +1012,1500,2.347197,1.0,2498.42,209847,32.04 +1013,1500,2.349528,1.0,2498.35,209853,32.04 +1014,1500,2.385504,1.0,2496.7,209991,32.06 +1015,1500,2.412254,1.0,2505.98,209214,31.94 +1016,1500,2.448342,1.0,2499.72,209739,32.02 +1017,1500,2.467616,1.0,2504.2,209363,31.96 +1018,1500,2.465448,1.0,2494.68,210162,32.09 +1019,1500,2.379567,1.0,2499.98,209716,32.02 +1020,1500,2.391365,1.0,2501.41,209596,32.0 +1021,1500,2.410615,1.0,2496.32,210024,32.07 +1022,1500,2.457156,1.0,2503.78,209398,31.97 +1023,1500,2.501823,1.0,2500.79,209649,32.01 +1024,1500,2.546373,1.0,2496.48,210010,32.06 +1025,1500,2.576119,1.0,2496.5,210008,32.06 +1026,1500,2.607259,1.0,2502.13,209536,31.99 +1027,1500,2.595531,1.0,3246.22,161507,24.66 +1028,1500,2.574413,1.0,2493.0,210303,32.11 +1029,1500,2.571716,1.0,2496.43,210015,32.06 +1030,1500,2.690301,1.0,2499.53,209754,32.02 +1031,1500,2.72224,1.0,2501.88,209557,31.99 +1032,1500,2.717924,1.0,2501.72,209570,32.0 +1033,1500,2.668257,1.0,2496.2,210034,32.07 +1034,1500,2.682798,1.0,2495.25,210114,32.08 +1035,1500,2.604531,1.0,2503.97,209382,31.97 +1036,1500,2.600583,1.0,2498.61,209832,32.04 +1037,1500,2.602779,1.0,2494.94,210140,32.08 +1038,1500,2.547325,1.0,2502.1,209539,31.99 +1039,1500,2.541426,1.0,2507.52,209086,31.92 +1040,1500,2.568093,1.0,2500.88,209641,32.01 +1041,1500,2.59226,1.0,2505.36,209266,31.95 +1042,1500,2.612483,1.0,2501.88,209557,31.99 +1043,1500,2.596725,1.0,2499.36,209768,32.03 +1044,1500,2.567148,1.0,2498.4,209849,32.04 +1045,1500,2.558828,1.0,2500.85,209644,32.01 +1046,1500,2.535682,1.0,2503.67,209408,31.97 +1047,1500,2.538711,1.0,2501.11,209622,32.0 +1048,1500,2.531904,1.0,2497.34,209938,32.05 +1049,1500,2.528915,1.0,2499.77,209734,32.02 +1050,1500,2.543039,1.0,2500.74,209652,32.01 +1051,1500,2.557352,1.0,2504.41,209346,31.96 +1052,1500,2.538566,1.0,2494.94,210140,32.08 +1053,1500,2.524232,1.0,2497.16,209953,32.05 +1054,1500,2.535897,1.0,2502.53,209503,31.99 +1055,1500,2.511785,1.0,2497.9,209891,32.05 +1056,1500,2.540567,1.0,2499.79,209733,32.02 +1057,1500,2.489501,1.0,2500.49,209673,32.01 +1058,1500,2.484187,1.0,2502.51,209504,31.99 +1059,1500,2.487643,1.0,2503.5,209421,31.97 +1060,1500,2.465699,1.0,2500.88,209641,32.01 +1061,1500,2.440185,1.0,2499.36,209768,32.03 +1062,1500,2.421542,1.0,2498.22,209864,32.04 +1063,1500,2.416919,1.0,2499.87,209725,32.02 +1064,1500,2.430497,1.0,2498.25,209862,32.04 +1065,1500,2.405876,1.0,2499.87,209725,32.02 +1066,1500,2.428573,1.0,2498.56,209836,32.04 +1067,1500,2.404212,1.0,2499.3,209774,32.03 +1068,1500,2.394564,1.0,2497.63,209914,32.05 +1069,1500,2.417578,1.0,2501.42,209596,32.0 +1070,1500,2.422496,1.0,2501.65,209576,32.0 +1071,1500,2.426493,1.0,2500.84,209644,32.01 +1072,1500,2.430037,1.0,2502.49,209506,31.99 +1073,1500,2.414623,1.0,2501.72,209571,32.0 +1074,1500,2.452738,1.0,2502.09,209540,31.99 +1075,1500,2.483931,1.0,2504.53,209336,31.96 +1076,1500,2.512765,1.0,2503.36,209434,31.98 +1077,1500,2.489914,1.0,2501.77,209566,32.0 +1078,1500,2.481657,1.0,2502.19,209531,31.99 +1079,1500,2.432451,1.0,2501.46,209593,32.0 +1080,1500,2.459475,1.0,2501.02,209629,32.01 +1081,1500,2.503668,1.0,2501.72,209570,32.0 +1082,1500,2.481614,1.0,2500.24,209694,32.02 +1083,1500,2.539859,1.0,2499.89,209724,32.02 +1084,1500,2.565516,1.0,2501.42,209596,32.0 +1085,1500,2.591217,1.0,2497.61,209915,32.05 +1086,1500,2.57702,1.0,2499.82,209730,32.02 +1087,1500,2.575139,1.0,2498.06,209878,32.04 +1088,1500,2.565641,1.0,2499.84,209728,32.02 +1089,1500,2.549572,1.0,2499.69,209741,32.02 +1090,1500,2.563537,1.0,2499.82,209730,32.02 +1091,1500,2.590664,1.0,2498.18,209868,32.04 +1092,1500,2.578195,1.0,2504.73,209319,31.96 +1093,1500,2.571242,1.0,2502.18,209532,31.99 +1094,1500,2.57999,1.0,2501.77,209567,32.0 +1095,1500,2.537517,1.0,2505.12,209286,31.95 +1096,1500,2.553105,1.0,2500.22,209696,32.02 +1097,1500,2.558843,1.0,2501.56,209584,32.0 +1098,1500,2.570094,1.0,2499.03,209796,32.03 +1099,1500,2.582346,1.0,2503.95,209384,31.97 +1100,1500,2.58644,1.0,2496.63,209997,32.06 +1101,1500,2.580091,1.0,2499.67,209742,32.02 +1102,1500,2.545139,1.0,2499.03,209796,32.03 +1103,1500,2.540063,1.0,2500.63,209661,32.01 +1104,1500,2.527436,1.0,2502.48,209506,31.99 +1105,1500,2.461445,1.0,2500.8,209647,32.01 +1106,1500,2.489434,1.0,2504.12,209370,31.97 +1107,1500,2.519811,1.0,2499.64,209745,32.02 +1108,1500,2.539433,1.0,3348.74,156562,23.9 +1109,1500,2.525969,1.0,2499.52,209755,32.02 +1110,1500,2.483961,1.0,2501.22,209613,32.0 +1111,1500,2.447209,1.0,2508.64,208992,31.91 +1112,1500,2.446468,1.0,2502.01,209546,31.99 +1113,1500,2.491553,1.0,2506.43,209177,31.94 +1114,1500,2.459502,1.0,2497.85,209895,32.05 +1115,1500,2.445792,1.0,2501.11,209622,32.0 +1116,1500,2.443582,1.0,2502.09,209540,31.99 +1117,1500,2.398858,1.0,2497.99,209883,32.04 +1118,1500,2.435793,1.0,2502.9,209472,31.98 +1119,1500,2.450739,1.0,2501.6,209580,32.0 +1120,1500,2.453562,1.0,2501.61,209580,32.0 +1121,1500,2.458797,1.0,2503.79,209397,31.97 +1122,1500,2.506607,1.0,2503.03,209461,31.98 +1123,1500,2.531235,1.0,2499.23,209779,32.03 +1124,1500,2.536931,1.0,2506.69,209155,31.93 +1125,1500,2.524225,1.0,2503.65,209409,31.97 +1126,1500,2.552255,1.0,2498.45,209845,32.04 +1127,1500,2.552342,1.0,2498.11,209873,32.04 +1128,1500,2.509837,1.0,2502.03,209544,31.99 +1129,1500,2.483425,1.0,2499.42,209764,32.03 +1130,1500,2.45853,1.0,2498.58,209834,32.04 +1131,1500,2.472561,1.0,2497.88,209893,32.05 +1132,1500,2.425508,1.0,2499.91,209722,32.02 +1133,1500,2.371697,1.0,2494.87,210146,32.08 +1134,1500,2.387674,1.0,2495.54,210090,32.08 +1135,1500,2.403613,1.0,2497.57,209919,32.05 +1136,1500,2.420161,1.0,2504.45,209342,31.96 +1137,1500,2.432826,1.0,2500.64,209661,32.01 +1138,1500,2.469571,1.0,2499.01,209798,32.03 +1139,1500,2.512893,1.0,2501.99,209548,31.99 +1140,1500,2.490199,1.0,2499.42,209763,32.03 +1141,1500,2.503078,1.0,2498.09,209875,32.04 +1142,1500,2.508764,1.0,2496.4,210017,32.06 +1143,1500,2.49701,1.0,2496.84,209980,32.06 +1144,1500,2.474529,1.0,2502.73,209486,31.98 +1145,1500,2.409986,1.0,2502.98,209465,31.98 +1146,1500,2.436207,1.0,2499.66,209743,32.02 +1147,1500,2.360981,1.0,2500.45,209677,32.01 +1148,1500,2.366878,1.0,2498.45,209845,32.04 +1149,1500,2.446126,1.0,2500.18,209699,32.02 +1150,1500,2.52981,1.0,2496.55,210004,32.06 +1151,1500,2.563946,1.0,2503.01,209462,31.98 +1152,1500,2.584806,1.0,2500.47,209675,32.01 +1153,1500,2.592696,1.0,2499.77,209734,32.02 +1154,1500,2.579082,1.0,2498.89,209808,32.03 +1155,1500,2.558637,1.0,2501.73,209570,32.0 +1156,1500,2.556868,1.0,2503.23,209444,31.98 +1157,1500,2.561193,1.0,2501.57,209583,32.0 +1158,1500,2.56733,1.0,2499.27,209776,32.03 +1159,1500,2.53404,1.0,2502.83,209477,31.98 +1160,1500,2.520394,1.0,2502.64,209493,31.98 +1161,1500,2.539431,1.0,2499.65,209744,32.02 +1162,1500,2.542988,1.0,2504.93,209302,31.96 +1163,1500,2.557495,1.0,2500.02,209713,32.02 +1164,1500,2.583253,1.0,2499.44,209762,32.03 +1165,1500,2.549616,1.0,2496.71,209991,32.06 +1166,1500,2.587962,1.0,2498.82,209813,32.03 +1167,1500,2.556298,1.0,2501.4,209598,32.0 +1168,1500,2.57664,1.0,2496.13,210040,32.07 +1169,1500,2.577357,1.0,2501.96,209550,31.99 +1170,1500,2.579842,1.0,2506.18,209197,31.94 +1171,1500,2.553053,1.0,2501.56,209584,32.0 +1172,1500,2.505223,1.0,2500.23,209696,32.02 +1173,1500,2.533689,1.0,2501.21,209614,32.0 +1174,1500,2.529751,1.0,2504.64,209326,31.96 +1175,1500,2.530845,1.0,2502.42,209512,31.99 +1176,1500,2.569716,1.0,2498.02,209881,32.04 +1177,1500,2.557234,1.0,2504.77,209316,31.96 +1178,1500,2.550491,1.0,2505.66,209241,31.95 +1179,1500,2.53948,1.0,2504.82,209311,31.96 +1180,1500,2.553053,1.0,2498.57,209835,32.04 +1181,1500,2.548203,1.0,2503.52,209420,31.97 +1182,1500,2.497362,1.0,2500.4,209681,32.01 +1183,1500,2.533041,1.0,2500.14,209703,32.02 +1184,1500,2.502415,1.0,2499.05,209794,32.03 +1185,1500,2.432266,1.0,2499.68,209742,32.02 +1186,1500,2.429734,1.0,2499.95,209719,32.02 +1187,1500,2.438783,1.0,2503.01,209462,31.98 +1188,1500,2.412814,1.0,2503.67,209407,31.97 +1189,1500,2.417956,1.0,3166.04,165597,25.28 +1190,1500,2.391732,1.0,2495.7,210076,32.07 +1191,1500,2.413218,1.0,2500.86,209643,32.01 +1192,1500,2.409355,1.0,2498.83,209813,32.03 +1193,1500,2.40385,1.0,2502.16,209534,31.99 +1194,1500,2.383438,1.0,2501.48,209590,32.0 +1195,1500,2.384559,1.0,2499.75,209735,32.02 +1196,1500,2.327422,1.0,2497.07,209961,32.06 +1197,1500,2.384903,1.0,2504.17,209365,31.96 +1198,1500,2.424343,1.0,2494.4,210186,32.09 +1199,1500,2.430876,1.0,2497.48,209926,32.05 +1200,1500,2.440996,1.0,2493.94,210224,32.1 +1201,1500,2.432617,1.0,2497.17,209952,32.05 +1202,1500,2.445783,1.0,2498.38,209851,32.04 +1203,1500,2.470045,1.0,2500.16,209702,32.02 +1204,1500,2.446939,1.0,2498.92,209806,32.03 +1205,1500,2.424146,1.0,2500.2,209698,32.02 +1206,1500,2.387269,1.0,2503.97,209382,31.97 +1207,1500,2.356603,1.0,2503.11,209454,31.98 +1208,1500,2.381411,1.0,2502.2,209531,31.99 +1209,1500,2.398901,1.0,2498.46,209844,32.04 +1210,1500,2.424929,1.0,2502.33,209519,31.99 +1211,1500,2.432458,1.0,2502.75,209484,31.98 +1212,1500,2.416121,1.0,2500.12,209704,32.02 +1213,1500,2.403905,1.0,2502.53,209503,31.99 +1214,1500,2.445069,1.0,2499.27,209776,32.03 +1215,1500,2.472159,1.0,2499.01,209798,32.03 +1216,1500,2.488684,1.0,2502.64,209493,31.98 +1217,1500,2.488064,1.0,2500.84,209644,32.01 +1218,1500,2.464704,1.0,2502.59,209497,31.99 +1219,1500,2.457899,1.0,2498.21,209865,32.04 +1220,1500,2.463797,1.0,2501.32,209604,32.0 +1221,1500,2.421569,1.0,2501.07,209625,32.0 +1222,1500,2.387758,1.0,2501.59,209582,32.0 +1223,1500,2.403879,1.0,2502.32,209521,31.99 +1224,1500,2.443035,1.0,2496.38,210019,32.06 +1225,1500,2.41826,1.0,2499.23,209779,32.03 +1226,1500,2.448829,1.0,2499.69,209741,32.02 +1227,1500,2.460668,1.0,2503.39,209431,31.97 +1228,1500,2.476715,1.0,2501.51,209588,32.0 +1229,1500,2.423908,1.0,2500.45,209677,32.01 +1230,1500,2.413463,1.0,2501.0,209631,32.01 +1231,1500,2.39471,1.0,2504.12,209369,31.97 +1232,1500,2.354343,1.0,2502.03,209544,31.99 +1233,1500,2.371003,1.0,2497.85,209895,32.05 +1234,1500,2.352386,1.0,2499.29,209774,32.03 +1235,1500,2.38966,1.0,2503.97,209382,31.97 +1236,1500,2.360726,1.0,2502.77,209483,31.98 +1237,1500,2.362755,1.0,2497.18,209951,32.05 +1238,1500,2.368698,1.0,2500.44,209678,32.01 +1239,1500,2.393627,1.0,2496.97,209969,32.06 +1240,1500,2.452025,1.0,2499.03,209796,32.03 +1241,1500,2.425301,1.0,2503.15,209451,31.98 +1242,1500,2.406693,1.0,2504.33,209352,31.96 +1243,1500,2.389649,1.0,2503.29,209439,31.98 +1244,1500,2.383802,1.0,2501.7,209572,32.0 +1245,1500,2.383217,1.0,2501.24,209611,32.0 +1246,1500,2.355744,1.0,2496.78,209985,32.06 +1247,1500,2.357943,1.0,2506.83,209144,31.93 +1248,1500,2.360747,1.0,2504.34,209351,31.96 +1249,1500,2.363689,1.0,2499.03,209796,32.03 +1250,1500,2.3612,1.0,2476.27,211724,32.32 +1251,1500,2.330253,1.0,2489.12,210631,32.16 +1252,1500,2.327421,1.0,2492.38,210355,32.12 +1253,1500,2.342858,1.0,2493.54,210258,32.1 +1254,1500,2.348323,1.0,2496.53,210006,32.06 +1255,1500,2.349545,1.0,2496.69,209993,32.06 +1256,1500,2.331776,1.0,2498.68,209826,32.04 +1257,1500,2.300347,1.0,2501.04,209628,32.0 +1258,1500,2.317687,1.0,2502.44,209510,31.99 +1259,1500,2.330505,1.0,2510.4,208846,31.89 +1260,1500,2.323666,1.0,2506.75,209150,31.93 +1261,1500,2.324743,1.0,2510.75,208817,31.88 +1262,1500,2.364188,1.0,2508.82,208977,31.91 +1263,1500,2.375115,1.0,2503.92,209386,31.97 +1264,1500,2.383553,1.0,2499.87,209726,32.02 +1265,1500,2.397808,1.0,2505.98,209214,31.94 +1266,1500,2.385561,1.0,2501.65,209577,32.0 +1267,1500,2.382359,1.0,2498.54,209837,32.04 +1268,1500,2.390636,1.0,2503.71,209404,31.97 +1269,1500,2.394031,1.0,2498.86,209811,32.03 +1270,1500,2.343721,1.0,3570.65,146832,22.42 +1271,1500,2.345929,1.0,2494.67,210163,32.09 +1272,1500,2.334842,1.0,2494.59,210170,32.09 +1273,1500,2.354259,1.0,2505.26,209274,31.95 +1274,1500,2.321031,1.0,2498.69,209825,32.03 +1275,1500,2.290331,1.0,2501.7,209572,32.0 +1276,1500,2.316285,1.0,2497.73,209906,32.05 +1277,1500,2.356199,1.0,2503.38,209432,31.97 +1278,1500,2.315412,1.0,2502.47,209507,31.99 +1279,1500,2.356142,1.0,2500.31,209689,32.01 +1280,1500,2.386494,1.0,2498.09,209875,32.04 +1281,1500,2.414202,1.0,2498.91,209806,32.03 +1282,1500,2.393893,1.0,2500.78,209649,32.01 +1283,1500,2.400295,1.0,2500.76,209651,32.01 +1284,1500,2.415913,1.0,2498.74,209820,32.03 +1285,1500,2.393784,1.0,2500.66,209660,32.01 +1286,1500,2.422404,1.0,2507.51,209087,31.92 +1287,1500,2.362867,1.0,2499.79,209732,32.02 +1288,1500,2.36999,1.0,2502.51,209504,31.99 +1289,1500,2.387106,1.0,2497.97,209885,32.04 +1290,1500,2.441367,1.0,2498.72,209822,32.03 +1291,1500,2.421063,1.0,2498.61,209831,32.04 +1292,1500,2.404441,1.0,2497.75,209904,32.05 +1293,1500,2.39602,1.0,2498.47,209843,32.04 +1294,1500,2.396105,1.0,2498.06,209877,32.04 +1295,1500,2.414203,1.0,2499.37,209768,32.03 +1296,1500,2.407398,1.0,2501.14,209619,32.0 +1297,1500,2.449826,1.0,2504.61,209329,31.96 +1298,1500,2.424689,1.0,2501.53,209586,32.0 +1299,1500,2.431997,1.0,2500.21,209697,32.02 +1300,1500,2.454086,1.0,2497.64,209913,32.05 +1301,1500,2.479593,1.0,2500.61,209663,32.01 +1302,1500,2.504229,1.0,2498.93,209805,32.03 +1303,1500,2.524538,1.0,2500.66,209660,32.01 +1304,1500,2.528756,1.0,2498.22,209864,32.04 +1305,1500,2.495952,1.0,2503.92,209386,31.97 +1306,1500,2.466344,1.0,2505.8,209230,31.94 +1307,1500,2.455489,1.0,2499.37,209767,32.03 +1308,1500,2.468556,1.0,2506.67,209157,31.93 +1309,1500,2.513764,1.0,2500.46,209676,32.01 +1310,1500,2.485792,1.0,2500.41,209680,32.01 +1311,1500,2.482397,1.0,2499.6,209748,32.02 +1312,1500,2.471948,1.0,2499.55,209752,32.02 +1313,1500,2.466685,1.0,2500.53,209670,32.01 +1314,1500,2.418181,1.0,2506.89,209138,31.93 +1315,1500,2.433514,1.0,2501.43,209595,32.0 +1316,1500,2.432581,1.0,2498.72,209822,32.03 +1317,1500,2.454954,1.0,2495.22,210117,32.08 +1318,1500,2.456899,1.0,2501.9,209556,31.99 +1319,1500,2.471288,1.0,2507.48,209089,31.92 +1320,1500,2.497319,1.0,2499.71,209739,32.02 +1321,1500,2.521943,1.0,2497.67,209910,32.05 +1322,1500,2.468437,1.0,2501.28,209607,32.0 +1323,1500,2.497319,1.0,2498.47,209843,32.04 +1324,1500,2.500568,1.0,2505.22,209278,31.95 +1325,1500,2.524526,1.0,2498.36,209853,32.04 +1326,1500,2.541899,1.0,2497.0,209966,32.06 +1327,1500,2.579647,1.0,2506.93,209135,31.93 +1328,1500,2.554086,1.0,2502.26,209525,31.99 +1329,1500,2.548375,1.0,2506.95,209133,31.93 +1330,1500,2.498161,1.0,2503.39,209431,31.97 +1331,1500,2.498972,1.0,2503.71,209404,31.97 +1332,1500,2.508089,1.0,2501.13,209620,32.0 +1333,1500,2.511404,1.0,2499.18,209783,32.03 +1334,1500,2.51094,1.0,2501.09,209624,32.0 +1335,1500,2.536113,1.0,2495.4,210101,32.08 +1336,1500,2.531636,1.0,2505.76,209233,31.94 +1337,1500,2.538024,1.0,2499.3,209773,32.03 +1338,1500,2.515331,1.0,2501.75,209568,32.0 +1339,1500,2.507357,1.0,2503.18,209448,31.98 +1340,1500,2.503695,1.0,2495.13,210124,32.08 +1341,1500,2.507368,1.0,2500.79,209648,32.01 +1342,1500,2.531401,1.0,2502.42,209512,31.99 +1343,1500,2.550755,1.0,2501.14,209619,32.0 +1344,1500,2.541343,1.0,2499.26,209777,32.03 +1345,1500,2.531321,1.0,2498.66,209827,32.04 +1346,1500,2.550525,1.0,2505.34,209267,31.95 +1347,1500,2.572877,1.0,2501.74,209569,32.0 +1348,1500,2.542516,1.0,2501.66,209575,32.0 +1349,1500,2.543491,1.0,2500.14,209703,32.02 +1350,1500,2.531649,1.0,2498.62,209830,32.04 +1351,1500,2.496322,1.0,3533.98,148356,22.65 +1352,1500,2.514389,1.0,2497.03,209964,32.06 +1353,1500,2.539929,1.0,2500.11,209706,32.02 +1354,1500,2.512404,1.0,2501.89,209556,31.99 +1355,1500,2.554807,1.0,2498.8,209816,32.03 +1356,1500,2.572568,1.0,2498.51,209840,32.04 +1357,1500,2.558059,1.0,2499.78,209733,32.02 +1358,1500,2.559718,1.0,2502.75,209484,31.98 +1359,1500,2.585213,1.0,2504.48,209340,31.96 +1360,1500,2.567475,1.0,2505.23,209277,31.95 +1361,1500,2.560613,1.0,2497.82,209898,32.05 +1362,1500,2.522151,1.0,2497.24,209947,32.05 +1363,1500,2.515407,1.0,2499.59,209749,32.02 +1364,1500,2.463668,1.0,2499.6,209749,32.02 +1365,1500,2.453724,1.0,2502.13,209536,31.99 +1366,1500,2.412913,1.0,2499.58,209750,32.02 +1367,1500,2.417648,1.0,2499.88,209725,32.02 +1368,1500,2.410528,1.0,2497.42,209932,32.05 +1369,1500,2.407004,1.0,2498.31,209857,32.04 +1370,1500,2.421171,1.0,2497.22,209948,32.05 +1371,1500,2.404136,1.0,2501.38,209599,32.0 +1372,1500,2.42543,1.0,2498.14,209870,32.04 +1373,1500,2.435518,1.0,2500.87,209642,32.01 +1374,1500,2.433409,1.0,2499.43,209763,32.03 +1375,1500,2.415074,1.0,2505.46,209258,31.95 +1376,1500,2.424119,1.0,2499.9,209723,32.02 +1377,1500,2.424437,1.0,2502.73,209486,31.98 +1378,1500,2.426031,1.0,2503.73,209402,31.97 +1379,1500,2.419726,1.0,2506.55,209167,31.93 +1380,1500,2.448142,1.0,2499.58,209750,32.02 +1381,1500,2.399573,1.0,2497.1,209958,32.06 +1382,1500,2.383089,1.0,2495.66,210079,32.07 +1383,1500,2.388769,1.0,2500.54,209670,32.01 +1384,1500,2.380467,1.0,2497.57,209919,32.05 +1385,1500,2.40175,1.0,2501.86,209558,31.99 +1386,1500,2.434829,1.0,2504.21,209362,31.96 +1387,1500,2.422441,1.0,2496.2,210034,32.07 +1388,1500,2.441884,1.0,2501.04,209627,32.0 +1389,1500,2.432226,1.0,2499.63,209745,32.02 +1390,1500,2.427714,1.0,2504.7,209321,31.96 +1391,1500,2.449333,1.0,2505.9,209221,31.94 +1392,1500,2.421657,1.0,2503.3,209438,31.98 +1393,1500,2.388789,1.0,2502.54,209502,31.99 +1394,1500,2.394633,1.0,2495.98,210053,32.07 +1395,1500,2.400391,1.0,2503.12,209453,31.98 +1396,1500,2.408024,1.0,2505.66,209241,31.95 +1397,1500,2.436805,1.0,2496.85,209979,32.06 +1398,1500,2.449819,1.0,2498.86,209810,32.03 +1399,1500,2.409758,1.0,2498.56,209836,32.04 +1400,1500,2.407156,1.0,2496.82,209981,32.06 +1401,1500,2.403219,1.0,2495.31,210109,32.08 +1402,1500,2.389298,1.0,2502.02,209545,31.99 +1403,1500,2.374202,1.0,2506.42,209178,31.94 +1404,1500,2.410423,1.0,2495.39,210102,32.08 +1405,1500,2.426619,1.0,2500.29,209691,32.01 +1406,1500,2.397991,1.0,2498.97,209801,32.03 +1407,1500,2.382572,1.0,2501.25,209610,32.0 +1408,1500,2.413051,1.0,2500.11,209706,32.02 +1409,1500,2.382012,1.0,2502.67,209491,31.98 +1410,1500,2.365499,1.0,2497.09,209959,32.06 +1411,1500,2.380447,1.0,2506.13,209202,31.94 +1412,1500,2.356123,1.0,2501.11,209621,32.0 +1413,1500,2.395847,1.0,2504.82,209311,31.96 +1414,1500,2.478718,1.0,2503.01,209463,31.98 +1415,1500,2.453347,1.0,2501.11,209621,32.0 +1416,1500,2.473875,1.0,2505.13,209285,31.95 +1417,1500,2.475766,1.0,2501.04,209627,32.0 +1418,1500,2.443631,1.0,2499.46,209760,32.03 +1419,1500,2.397742,1.0,2499.47,209759,32.02 +1420,1500,2.433936,1.0,2496.71,209991,32.06 +1421,1500,2.418765,1.0,2502.3,209522,31.99 +1422,1500,2.397046,1.0,2497.44,209929,32.05 +1423,1500,2.391084,1.0,2499.23,209779,32.03 +1424,1500,2.398878,1.0,2498.95,209803,32.03 +1425,1500,2.419595,1.0,2499.91,209723,32.02 +1426,1500,2.45759,1.0,2502.04,209544,31.99 +1427,1500,2.493086,1.0,2502.18,209532,31.99 +1428,1500,2.485929,1.0,2498.83,209812,32.03 +1429,1500,2.487202,1.0,2499.08,209792,32.03 +1430,1500,2.471029,1.0,2502.32,209520,31.99 +1431,1500,2.456697,1.0,2505.03,209294,31.95 +1432,1500,2.437899,1.0,2747.31,190836,29.14 +1433,1500,2.431286,1.0,2500.94,209636,32.01 +1434,1500,2.451353,1.0,2500.11,209706,32.02 +1435,1500,2.444935,1.0,2507.48,209089,31.92 +1436,1500,2.460388,1.0,2500.79,209648,32.01 +1437,1500,2.473801,1.0,2506.7,209154,31.93 +1438,1500,2.483231,1.0,2501.54,209586,32.0 +1439,1500,2.443377,1.0,2502.14,209536,31.99 +1440,1500,2.462744,1.0,2504.65,209325,31.96 +1441,1500,2.44869,1.0,2502.88,209473,31.98 +1442,1500,2.443707,1.0,2497.9,209891,32.05 +1443,1500,2.447738,1.0,2503.93,209386,31.97 +1444,1500,2.430013,1.0,2498.28,209859,32.04 +1445,1500,2.42855,1.0,2503.7,209405,31.97 +1446,1500,2.428568,1.0,2501.13,209620,32.0 +1447,1500,2.453339,1.0,2501.42,209596,32.0 +1448,1500,2.472597,1.0,2503.4,209430,31.97 +1449,1500,2.449282,1.0,2499.85,209727,32.02 +1450,1500,2.456129,1.0,2505.5,209254,31.95 +1451,1500,2.478262,1.0,2495.35,210105,32.08 +1452,1500,2.469788,1.0,2500.53,209670,32.01 +1453,1500,2.490518,1.0,2507.76,209066,31.92 +1454,1500,2.480713,1.0,2501.42,209595,32.0 +1455,1500,2.483244,1.0,2500.85,209643,32.01 +1456,1500,2.475816,1.0,2499.12,209789,32.03 +1457,1500,2.505269,1.0,2502.45,209509,31.99 +1458,1500,2.40804,1.0,2500.37,209684,32.01 +1459,1500,2.404778,1.0,2501.87,209558,31.99 +1460,1500,2.401158,1.0,2505.31,209270,31.95 +1461,1500,2.39326,1.0,2499.41,209764,32.03 +1462,1500,2.377952,1.0,2501.03,209628,32.01 +1463,1500,2.363887,1.0,2497.56,209920,32.05 +1464,1500,2.30043,1.0,2496.16,210038,32.07 +1465,1500,2.296305,1.0,2504.82,209311,31.96 +1466,1500,2.294729,1.0,2505.22,209278,31.95 +1467,1500,2.273269,1.0,2500.66,209659,32.01 +1468,1500,2.2917,1.0,2498.76,209819,32.03 +1469,1500,2.234252,1.0,2499.27,209776,32.03 +1470,1500,2.249146,1.0,2503.34,209435,31.98 +1471,1500,2.286775,1.0,2504.94,209301,31.96 +1472,1500,2.296955,1.0,2497.88,209892,32.05 +1473,1500,2.349969,1.0,2500.89,209640,32.01 +1474,1500,2.382145,1.0,2502.49,209506,31.99 +1475,1500,2.374149,1.0,2497.38,209935,32.05 +1476,1500,2.375506,1.0,2501.69,209573,32.0 +1477,1500,2.38793,1.0,2501.36,209601,32.0 +1478,1500,2.390028,1.0,2499.69,209741,32.02 +1479,1500,2.376601,1.0,2499.56,209752,32.02 +1480,1500,2.330563,1.0,2504.31,209353,31.96 +1481,1500,2.326115,1.0,2498.09,209875,32.04 +1482,1500,2.331603,1.0,2500.58,209666,32.01 +1483,1500,2.368139,1.0,2503.89,209389,31.97 +1484,1500,2.375422,1.0,2498.38,209850,32.04 +1485,1500,2.354445,1.0,2502.87,209475,31.98 +1486,1500,2.357608,1.0,2499.33,209771,32.03 +1487,1500,2.380119,1.0,2502.01,209546,31.99 +1488,1500,2.400317,1.0,2500.08,209708,32.02 +1489,1500,2.360456,1.0,2500.68,209658,32.01 +1490,1500,2.400154,1.0,2504.74,209317,31.96 +1491,1500,2.391098,1.0,2508.42,209011,31.91 +1492,1500,2.382892,1.0,2498.7,209824,32.03 +1493,1500,2.386637,1.0,2501.3,209606,32.0 +1494,1500,2.416373,1.0,2503.63,209411,31.97 +1495,1500,2.39764,1.0,2505.17,209282,31.95 +1496,1500,2.419714,1.0,2499.4,209765,32.03 +1497,1500,2.435781,1.0,2501.58,209582,32.0 +1498,1500,2.414833,1.0,2502.76,209483,31.98 +1499,1500,2.365202,1.0,2500.78,209649,32.01 diff --git a/logs/d32cmap_provenance.json b/logs/d32cmap_provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..8bc73cda5f3d11191128d29df9dddbdf6f14ed64 --- /dev/null +++ b/logs/d32cmap_provenance.json @@ -0,0 +1,23 @@ +{ + "run_tag": "d32cmap", + "origin": "karpathy/nanochat-d32:model_000650.pt", + "source_repo": "karpathy/nanochat-d32", + "source_model": "model_000650.pt", + "attention": "HMAP alpha=0.0 (AMAP)", + "window_pattern": "L", + "depth": 32, + "resumed_step": 0, + "horizon": 1500, + "new_steps_requested": 1500, + "num_gpus": 8, + "data_shards_requested": 240, + "extra_args": "--attn-variant=hmap --hmap-alpha=1.0 --hmap-beta=1.0 --device-batch-size=2 --total-batch-size=524288 --embedding-lr=0.03 --unembedding-lr=0.0008 --matrix-lr=0.002 --scalar-lr=0.05 --weight-decay=0.0 --warmup-steps=40 --warmdown-ratio=0.0 --final-lr-frac=1.0 --core-metric-every=-1 --eval-every=100 --sample-every=250", + "graft_neutralization": { + "value_embeds": "zeroed", + "resid_lambdas": 1.0, + "x0_lambdas": 0.0, + "backout_lambda": 0.0, + "smear_lambda": "init (0.0)", + "ve_gate/smear_gate": "init (inert via zeroed tables / smear_lambda=0)" + } +} diff --git a/logs/d32cmap_train.log b/logs/d32cmap_train.log new file mode 100644 index 0000000000000000000000000000000000000000..20a4e1eee83e62cb1d1cde4e85f7e147855831cd --- /dev/null +++ b/logs/d32cmap_train.log @@ -0,0 +1,1761 @@ + + █████ █████ + ░░███ ░░███ + ████████ ██████ ████████ ██████ ██████ ░███████ ██████ ███████ + ░░███░░███ ░░░░░███ ░░███░░███ ███░░███ ███░░███ ░███░░███ ░░░░░███░░░███░ + ░███ ░███ ███████ ░███ ░███ ░███ ░███░███ ░░░ ░███ ░███ ███████ ░███ + ░███ ░███ ███░░███ ░███ ░███ ░███ ░███░███ ███ ░███ ░███ ███░░███ ░███ ███ + ████ █████░░████████ ████ █████░░██████ ░░██████ ████ █████░░███████ ░░█████ + ░░░░ ░░░░░ ░░░░░░░░ ░░░░ ░░░░░ ░░░░░░ ░░░░░░ ░░░░ ░░░░░ ░░░░░░░░ ░░░░░ + +Autodetected device type: cuda +2026-08-10 21:12:28,190 - nanochat.common - INFO - Distributed world size: 8 +GPU: NVIDIA H200 | Peak FLOPS (BF16): 9.89e+14 +COMPUTE_DTYPE: torch.bfloat16 (auto-detected: CUDA SM 90 (bf16 supported)) +✓ Using Flash Attention 3: efficient, new and awesome. +Vocab size: 65,536 +Model config: +{ + "sequence_len": 2048, + "vocab_size": 65536, + "n_layer": 32, + "n_head": 16, + "n_kv_head": 16, + "n_embd": 2048, + "window_pattern": "L", + "attn_variant": "hmap", + "hmap_alpha": 1.0, + "hmap_beta": 1.0, + "witten": false, + "bidirectional": false +} +[d32ft] Weights-only warm start from /root/.cache/nanochat-d32ft/bootstrap/karpathy-d32/model_000650.pt +[d32ft] New optimizer + new dataloader state; local step starts at 0 +[d32ft] missing from checkpoint: backout_lambda +[d32ft] missing from checkpoint: resid_lambdas +[d32ft] missing from checkpoint: smear_gate.weight +[d32ft] missing from checkpoint: smear_lambda +[d32ft] missing from checkpoint: transformer.h.1.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.11.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.13.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.15.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.17.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.19.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.21.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.23.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.25.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.27.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.29.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.3.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.31.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.5.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.7.attn.ve_gate.weight +[d32ft] missing from checkpoint: transformer.h.9.attn.ve_gate.weight +[d32ft] missing from checkpoint: value_embeds.1.weight +[d32ft] missing from checkpoint: value_embeds.11.weight +[d32ft] missing from checkpoint: value_embeds.13.weight +[d32ft] missing from checkpoint: value_embeds.15.weight +[d32ft] missing from checkpoint: value_embeds.17.weight +[d32ft] missing from checkpoint: value_embeds.19.weight +[d32ft] missing from checkpoint: value_embeds.21.weight +[d32ft] missing from checkpoint: value_embeds.23.weight +[d32ft] missing from checkpoint: value_embeds.25.weight +[d32ft] missing from checkpoint: value_embeds.27.weight +[d32ft] missing from checkpoint: value_embeds.29.weight +[d32ft] missing from checkpoint: value_embeds.3.weight +[d32ft] missing from checkpoint: value_embeds.31.weight +[d32ft] missing from checkpoint: value_embeds.5.weight +[d32ft] missing from checkpoint: value_embeds.7.weight +[d32ft] missing from checkpoint: value_embeds.9.weight +[d32ft] missing from checkpoint: x0_lambdas +[d32ft] graft-neutral override: resid_lambdas -> 1.0 (was from-scratch init ~1.13) +[d32ft] graft-neutral override: x0_lambdas -> 0.0 (was from-scratch init) +[d32ft] kept init (verify neutral!): smear_lambda shape=(1,) mean=0.0000 values=[0.000] +[d32ft] graft-neutral override: backout_lambda -> 0.0 (was from-scratch init) +[d32ft] kept init (verify neutral!): transformer.h.1.attn.ve_gate.weight shape=(16, 12) mean=0.0098 values=[0.014, 0.000, 0.015, 0.001, 0.002, 0.010, 0.009, 0.010, ...] +[d32ft] kept init (verify neutral!): transformer.h.3.attn.ve_gate.weight shape=(16, 12) mean=0.0101 values=[0.007, 0.005, 0.002, 0.012, 0.010, 0.002, 0.018, 0.012, ...] +[d32ft] kept init (verify neutral!): transformer.h.5.attn.ve_gate.weight shape=(16, 12) mean=0.0101 values=[0.003, 0.019, 0.008, 0.014, 0.018, 0.019, 0.007, 0.013, ...] +[d32ft] kept init (verify neutral!): transformer.h.7.attn.ve_gate.weight shape=(16, 12) mean=0.0096 values=[0.002, 0.010, 0.006, 0.009, 0.003, 0.007, 0.018, 0.005, ...] +[d32ft] kept init (verify neutral!): transformer.h.9.attn.ve_gate.weight shape=(16, 12) mean=0.0097 values=[0.013, 0.015, 0.014, 0.018, 0.001, 0.017, 0.009, 0.017, ...] +[d32ft] kept init (verify neutral!): transformer.h.11.attn.ve_gate.weight shape=(16, 12) mean=0.0100 values=[0.008, 0.004, 0.007, 0.014, 0.013, 0.009, 0.003, 0.003, ...] +[d32ft] kept init (verify neutral!): transformer.h.13.attn.ve_gate.weight shape=(16, 12) mean=0.0102 values=[0.000, 0.003, 0.004, 0.005, 0.008, 0.018, 0.011, 0.008, ...] +[d32ft] kept init (verify neutral!): transformer.h.15.attn.ve_gate.weight shape=(16, 12) mean=0.0103 values=[0.017, 0.008, 0.002, 0.014, 0.001, 0.011, 0.003, 0.011, ...] +[d32ft] kept init (verify neutral!): transformer.h.17.attn.ve_gate.weight shape=(16, 12) mean=0.0098 values=[0.012, 0.008, 0.019, 0.007, 0.014, 0.006, 0.002, 0.005, ...] +[d32ft] kept init (verify neutral!): transformer.h.19.attn.ve_gate.weight shape=(16, 12) mean=0.0107 values=[0.010, 0.010, 0.020, 0.014, 0.019, 0.019, 0.003, 0.006, ...] +[d32ft] kept init (verify neutral!): transformer.h.21.attn.ve_gate.weight shape=(16, 12) mean=0.0095 values=[0.007, 0.000, 0.016, 0.005, 0.014, 0.010, 0.003, 0.003, ...] +[d32ft] kept init (verify neutral!): transformer.h.23.attn.ve_gate.weight shape=(16, 12) mean=0.0099 values=[0.019, 0.017, 0.020, 0.001, 0.009, 0.018, 0.013, 0.006, ...] +[d32ft] kept init (verify neutral!): transformer.h.25.attn.ve_gate.weight shape=(16, 12) mean=0.0099 values=[0.013, 0.014, 0.010, 0.004, 0.005, 0.004, 0.009, 0.002, ...] +[d32ft] kept init (verify neutral!): transformer.h.27.attn.ve_gate.weight shape=(16, 12) mean=0.0099 values=[0.003, 0.005, 0.013, 0.002, 0.010, 0.009, 0.009, 0.012, ...] +[d32ft] kept init (verify neutral!): transformer.h.29.attn.ve_gate.weight shape=(16, 12) mean=0.0104 values=[0.016, 0.014, 0.018, 0.018, 0.007, 0.018, 0.019, 0.002, ...] +[d32ft] kept init (verify neutral!): transformer.h.31.attn.ve_gate.weight shape=(16, 12) mean=0.0098 values=[0.019, 0.011, 0.020, 0.008, 0.017, 0.020, 0.001, 0.008, ...] +[d32ft] kept init (verify neutral!): smear_gate.weight shape=(1, 24) mean=0.0102 values=[0.008, 0.013, 0.017, 0.004, 0.015, 0.017, 0.017, 0.002, ...] +[d32ft] zero-initialized VE table: value_embeds.1.weight +[d32ft] zero-initialized VE table: value_embeds.3.weight +[d32ft] zero-initialized VE table: value_embeds.5.weight +[d32ft] zero-initialized VE table: value_embeds.7.weight +[d32ft] zero-initialized VE table: value_embeds.9.weight +[d32ft] zero-initialized VE table: value_embeds.11.weight +[d32ft] zero-initialized VE table: value_embeds.13.weight +[d32ft] zero-initialized VE table: value_embeds.15.weight +[d32ft] zero-initialized VE table: value_embeds.17.weight +[d32ft] zero-initialized VE table: value_embeds.19.weight +[d32ft] zero-initialized VE table: value_embeds.21.weight +[d32ft] zero-initialized VE table: value_embeds.23.weight +[d32ft] zero-initialized VE table: value_embeds.25.weight +[d32ft] zero-initialized VE table: value_embeds.27.weight +[d32ft] zero-initialized VE table: value_embeds.29.weight +[d32ft] zero-initialized VE table: value_embeds.31.weight +Parameter counts: +wte : 134,217,728 +value_embeds : 2,147,483,648 +lm_head : 134,217,728 +transformer_matrices : 1,610,615,808 +scalars : 90 +total : 4,026,535,002 +Estimated FLOPs per token: 1.207961e+10 +Scaling the LR for the AdamW parameters ∝1/√(2048/768) = 0.612372 +Using user-provided number of iterations: 1,500 +Total number of training tokens: 786,432,000 +Tokens : Scaling params ratio: 0.45 +Total training FLOPs estimate: 9.499795e+18 +Tokens / micro-batch / rank: 2 x 2048 = 4,096 +Tokens / micro-batch: 32,768 +Total batch size 524,288 => gradient accumulation steps: 16 +Step 00000 | Validation bpb: 1.795972 +Step 00000 | ICL early: 5.1304 late: 5.8076 score: +0.6772 | Induction loss: 15.0148 acc: 0.0000 (random-half: 15.1274) +step 00000/01500 (0.00%) | loss: 5.939279 | lrm: 0.03 | dt: 173819.10ms | tok/sec: 3,016 | bf16_mfu: 0.46 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00001/01500 (0.07%) | loss: 5.956664 | lrm: 0.05 | dt: 2685.29ms | tok/sec: 195,244 | bf16_mfu: 29.81 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00002/01500 (0.13%) | loss: 5.883790 | lrm: 0.07 | dt: 2861.30ms | tok/sec: 183,234 | bf16_mfu: 27.98 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00003/01500 (0.20%) | loss: 5.876734 | lrm: 0.10 | dt: 2482.99ms | tok/sec: 211,152 | bf16_mfu: 32.24 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00004/01500 (0.27%) | loss: 5.805432 | lrm: 0.12 | dt: 2494.62ms | tok/sec: 210,167 | bf16_mfu: 32.09 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00005/01500 (0.33%) | loss: 5.792202 | lrm: 0.15 | dt: 2494.67ms | tok/sec: 210,162 | bf16_mfu: 32.09 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00006/01500 (0.40%) | loss: 5.767968 | lrm: 0.17 | dt: 2502.11ms | tok/sec: 209,538 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00007/01500 (0.47%) | loss: 5.776086 | lrm: 0.20 | dt: 2500.57ms | tok/sec: 209,667 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00008/01500 (0.53%) | loss: 5.763400 | lrm: 0.23 | dt: 2490.52ms | tok/sec: 210,513 | bf16_mfu: 32.14 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00009/01500 (0.60%) | loss: 5.676017 | lrm: 0.25 | dt: 2494.58ms | tok/sec: 210,170 | bf16_mfu: 32.09 | epoch: 1 pq: 0 rg: 8 | total time: 0.00m +step 00010/01500 (0.67%) | loss: 5.621683 | lrm: 0.28 | dt: 2497.56ms | tok/sec: 209,919 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 16 | total time: 0.00m +step 00011/01500 (0.73%) | loss: 5.571587 | lrm: 0.30 | dt: 2499.24ms | tok/sec: 209,779 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 16 | total time: 0.04m | eta: 62.0m +step 00012/01500 (0.80%) | loss: 5.548217 | lrm: 0.33 | dt: 2498.47ms | tok/sec: 209,843 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 16 | total time: 0.08m | eta: 62.0m +step 00013/01500 (0.87%) | loss: 5.470431 | lrm: 0.35 | dt: 2498.29ms | tok/sec: 209,858 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 16 | total time: 0.12m | eta: 61.9m +step 00014/01500 (0.93%) | loss: 5.430450 | lrm: 0.38 | dt: 2501.25ms | tok/sec: 209,610 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 16 | total time: 0.17m | eta: 61.9m +step 00015/01500 (1.00%) | loss: 5.347769 | lrm: 0.40 | dt: 2499.86ms | tok/sec: 209,726 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 16 | total time: 0.21m | eta: 61.9m +step 00016/01500 (1.07%) | loss: 5.246905 | lrm: 0.42 | dt: 2499.73ms | tok/sec: 209,737 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 16 | total time: 0.25m | eta: 61.8m +step 00017/01500 (1.13%) | loss: 5.172430 | lrm: 0.45 | dt: 2493.68ms | tok/sec: 210,246 | bf16_mfu: 32.10 | epoch: 1 pq: 0 rg: 16 | total time: 0.29m | eta: 61.8m +step 00018/01500 (1.20%) | loss: 5.120843 | lrm: 0.47 | dt: 2499.59ms | tok/sec: 209,749 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 24 | total time: 0.33m | eta: 61.7m +step 00019/01500 (1.27%) | loss: 5.091685 | lrm: 0.50 | dt: 2494.79ms | tok/sec: 210,153 | bf16_mfu: 32.09 | epoch: 1 pq: 0 rg: 24 | total time: 0.37m | eta: 61.7m +step 00020/01500 (1.33%) | loss: 5.031218 | lrm: 0.53 | dt: 2497.78ms | tok/sec: 209,901 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 24 | total time: 0.42m | eta: 61.6m +step 00021/01500 (1.40%) | loss: 4.924438 | lrm: 0.55 | dt: 2500.80ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 24 | total time: 0.46m | eta: 61.6m +step 00022/01500 (1.47%) | loss: 4.878394 | lrm: 0.57 | dt: 2500.97ms | tok/sec: 209,633 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 24 | total time: 0.50m | eta: 61.6m +step 00023/01500 (1.53%) | loss: 4.790773 | lrm: 0.60 | dt: 2500.81ms | tok/sec: 209,647 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 24 | total time: 0.54m | eta: 61.5m +step 00024/01500 (1.60%) | loss: 4.750170 | lrm: 0.62 | dt: 2502.89ms | tok/sec: 209,473 | bf16_mfu: 31.98 | epoch: 1 pq: 0 rg: 24 | total time: 0.58m | eta: 61.5m +step 00025/01500 (1.67%) | loss: 4.698532 | lrm: 0.65 | dt: 2500.15ms | tok/sec: 209,703 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 24 | total time: 0.62m | eta: 61.4m +step 00026/01500 (1.73%) | loss: 4.655308 | lrm: 0.68 | dt: 2497.32ms | tok/sec: 209,939 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 32 | total time: 0.67m | eta: 61.4m +step 00027/01500 (1.80%) | loss: 4.655427 | lrm: 0.70 | dt: 2501.75ms | tok/sec: 209,568 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 32 | total time: 0.71m | eta: 61.4m +step 00028/01500 (1.87%) | loss: 4.569183 | lrm: 0.72 | dt: 2500.24ms | tok/sec: 209,695 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 32 | total time: 0.75m | eta: 61.3m +step 00029/01500 (1.93%) | loss: 4.494453 | lrm: 0.75 | dt: 2495.72ms | tok/sec: 210,074 | bf16_mfu: 32.07 | epoch: 1 pq: 0 rg: 32 | total time: 0.79m | eta: 61.3m +step 00030/01500 (2.00%) | loss: 4.472533 | lrm: 0.78 | dt: 2500.82ms | tok/sec: 209,646 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 32 | total time: 0.83m | eta: 61.2m +step 00031/01500 (2.07%) | loss: 4.447625 | lrm: 0.80 | dt: 2502.08ms | tok/sec: 209,541 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 32 | total time: 0.87m | eta: 61.2m +step 00032/01500 (2.13%) | loss: 4.363409 | lrm: 0.82 | dt: 2502.24ms | tok/sec: 209,527 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 32 | total time: 0.92m | eta: 61.2m +step 00033/01500 (2.20%) | loss: 4.322013 | lrm: 0.85 | dt: 2499.29ms | tok/sec: 209,775 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 32 | total time: 0.96m | eta: 61.1m +step 00034/01500 (2.27%) | loss: 4.241499 | lrm: 0.88 | dt: 2501.64ms | tok/sec: 209,577 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 40 | total time: 1.00m | eta: 61.1m +step 00035/01500 (2.33%) | loss: 4.149794 | lrm: 0.90 | dt: 2498.59ms | tok/sec: 209,833 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 40 | total time: 1.04m | eta: 61.0m +step 00036/01500 (2.40%) | loss: 4.086062 | lrm: 0.93 | dt: 2500.66ms | tok/sec: 209,659 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 40 | total time: 1.08m | eta: 61.0m +step 00037/01500 (2.47%) | loss: 4.060564 | lrm: 0.95 | dt: 2495.85ms | tok/sec: 210,063 | bf16_mfu: 32.07 | epoch: 1 pq: 0 rg: 40 | total time: 1.12m | eta: 60.9m +step 00038/01500 (2.53%) | loss: 4.015661 | lrm: 0.97 | dt: 2498.20ms | tok/sec: 209,866 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 40 | total time: 1.17m | eta: 60.9m +step 00039/01500 (2.60%) | loss: 3.966935 | lrm: 1.00 | dt: 2496.40ms | tok/sec: 210,017 | bf16_mfu: 32.06 | epoch: 1 pq: 0 rg: 40 | total time: 1.21m | eta: 60.9m +step 00040/01500 (2.67%) | loss: 3.943025 | lrm: 1.00 | dt: 2495.80ms | tok/sec: 210,067 | bf16_mfu: 32.07 | epoch: 1 pq: 0 rg: 40 | total time: 1.25m | eta: 60.8m +step 00041/01500 (2.73%) | loss: 3.921660 | lrm: 1.00 | dt: 2504.60ms | tok/sec: 209,329 | bf16_mfu: 31.96 | epoch: 1 pq: 0 rg: 48 | total time: 1.29m | eta: 60.8m +step 00042/01500 (2.80%) | loss: 3.918710 | lrm: 1.00 | dt: 2497.40ms | tok/sec: 209,933 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 48 | total time: 1.33m | eta: 60.7m +step 00043/01500 (2.87%) | loss: 3.887281 | lrm: 1.00 | dt: 2502.83ms | tok/sec: 209,477 | bf16_mfu: 31.98 | epoch: 1 pq: 0 rg: 48 | total time: 1.37m | eta: 60.7m +step 00044/01500 (2.93%) | loss: 3.897078 | lrm: 1.00 | dt: 2499.14ms | tok/sec: 209,787 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 48 | total time: 1.42m | eta: 60.7m +step 00045/01500 (3.00%) | loss: 3.824480 | lrm: 1.00 | dt: 2500.70ms | tok/sec: 209,656 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 48 | total time: 1.46m | eta: 60.6m +step 00046/01500 (3.07%) | loss: 3.721530 | lrm: 1.00 | dt: 2497.85ms | tok/sec: 209,895 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 48 | total time: 1.50m | eta: 60.6m +step 00047/01500 (3.13%) | loss: 3.684740 | lrm: 1.00 | dt: 2500.95ms | tok/sec: 209,635 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 48 | total time: 1.54m | eta: 60.5m +step 00048/01500 (3.20%) | loss: 3.662864 | lrm: 1.00 | dt: 2497.91ms | tok/sec: 209,890 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 48 | total time: 1.58m | eta: 60.5m +step 00049/01500 (3.27%) | loss: 3.596936 | lrm: 1.00 | dt: 2497.21ms | tok/sec: 209,949 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 56 | total time: 1.62m | eta: 60.4m +step 00050/01500 (3.33%) | loss: 3.584375 | lrm: 1.00 | dt: 2501.48ms | tok/sec: 209,591 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 56 | total time: 1.67m | eta: 60.4m +step 00051/01500 (3.40%) | loss: 3.537943 | lrm: 1.00 | dt: 2501.91ms | tok/sec: 209,555 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 56 | total time: 1.71m | eta: 60.4m +step 00052/01500 (3.47%) | loss: 3.532923 | lrm: 1.00 | dt: 2494.04ms | tok/sec: 210,216 | bf16_mfu: 32.09 | epoch: 1 pq: 0 rg: 56 | total time: 1.75m | eta: 60.3m +step 00053/01500 (3.53%) | loss: 3.510258 | lrm: 1.00 | dt: 2500.43ms | tok/sec: 209,679 | bf16_mfu: 32.01 | epoch: 1 pq: 0 rg: 56 | total time: 1.79m | eta: 60.3m +step 00054/01500 (3.60%) | loss: 3.510891 | lrm: 1.00 | dt: 2497.17ms | tok/sec: 209,952 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 56 | total time: 1.83m | eta: 60.2m +step 00055/01500 (3.67%) | loss: 3.529868 | lrm: 1.00 | dt: 2499.21ms | tok/sec: 209,781 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 56 | total time: 1.87m | eta: 60.2m +step 00056/01500 (3.73%) | loss: 3.606472 | lrm: 1.00 | dt: 2502.32ms | tok/sec: 209,521 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 64 | total time: 1.92m | eta: 60.2m +step 00057/01500 (3.80%) | loss: 3.568959 | lrm: 1.00 | dt: 2497.56ms | tok/sec: 209,919 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 64 | total time: 1.96m | eta: 60.1m +step 00058/01500 (3.87%) | loss: 3.569934 | lrm: 1.00 | dt: 2496.95ms | tok/sec: 209,971 | bf16_mfu: 32.06 | epoch: 1 pq: 0 rg: 64 | total time: 2.00m | eta: 60.1m +step 00059/01500 (3.93%) | loss: 3.554943 | lrm: 1.00 | dt: 2496.78ms | tok/sec: 209,985 | bf16_mfu: 32.06 | epoch: 1 pq: 0 rg: 64 | total time: 2.04m | eta: 60.0m +step 00060/01500 (4.00%) | loss: 3.484738 | lrm: 1.00 | dt: 2496.90ms | tok/sec: 209,975 | bf16_mfu: 32.06 | epoch: 1 pq: 0 rg: 64 | total time: 2.08m | eta: 60.0m +step 00061/01500 (4.07%) | loss: 3.424122 | lrm: 1.00 | dt: 2496.32ms | tok/sec: 210,024 | bf16_mfu: 32.07 | epoch: 1 pq: 0 rg: 64 | total time: 2.12m | eta: 59.9m +step 00062/01500 (4.13%) | loss: 3.383661 | lrm: 1.00 | dt: 2502.71ms | tok/sec: 209,488 | bf16_mfu: 31.98 | epoch: 1 pq: 0 rg: 64 | total time: 2.17m | eta: 59.9m +step 00063/01500 (4.20%) | loss: 3.327162 | lrm: 1.00 | dt: 2498.35ms | tok/sec: 209,854 | bf16_mfu: 32.04 | epoch: 1 pq: 0 rg: 64 | total time: 2.21m | eta: 59.9m +step 00064/01500 (4.27%) | loss: 3.325502 | lrm: 1.00 | dt: 2501.26ms | tok/sec: 209,609 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 72 | total time: 2.25m | eta: 59.8m +step 00065/01500 (4.33%) | loss: 3.307940 | lrm: 1.00 | dt: 2499.18ms | tok/sec: 209,783 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 72 | total time: 2.29m | eta: 59.8m +step 00066/01500 (4.40%) | loss: 3.332836 | lrm: 1.00 | dt: 2500.05ms | tok/sec: 209,711 | bf16_mfu: 32.02 | epoch: 1 pq: 0 rg: 72 | total time: 2.33m | eta: 59.7m +step 00067/01500 (4.47%) | loss: 3.303473 | lrm: 1.00 | dt: 2494.88ms | tok/sec: 210,145 | bf16_mfu: 32.08 | epoch: 1 pq: 0 rg: 72 | total time: 2.37m | eta: 59.7m +step 00068/01500 (4.53%) | loss: 3.301317 | lrm: 1.00 | dt: 2499.16ms | tok/sec: 209,785 | bf16_mfu: 32.03 | epoch: 1 pq: 0 rg: 72 | total time: 2.42m | eta: 59.6m +step 00069/01500 (4.60%) | loss: 3.300453 | lrm: 1.00 | dt: 2497.43ms | tok/sec: 209,931 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 72 | total time: 2.46m | eta: 59.6m +step 00070/01500 (4.67%) | loss: 3.303051 | lrm: 1.00 | dt: 2503.44ms | tok/sec: 209,427 | bf16_mfu: 31.97 | epoch: 1 pq: 0 rg: 72 | total time: 2.50m | eta: 59.6m +step 00071/01500 (4.73%) | loss: 3.346494 | lrm: 1.00 | dt: 2647.30ms | tok/sec: 198,046 | bf16_mfu: 30.24 | epoch: 1 pq: 0 rg: 80 | total time: 2.54m | eta: 59.6m +step 00072/01500 (4.80%) | loss: 3.331093 | lrm: 1.00 | dt: 2501.62ms | tok/sec: 209,579 | bf16_mfu: 32.00 | epoch: 1 pq: 0 rg: 80 | total time: 2.58m | eta: 59.5m +step 00073/01500 (4.87%) | loss: 3.307411 | lrm: 1.00 | dt: 2497.78ms | tok/sec: 209,901 | bf16_mfu: 32.05 | epoch: 1 pq: 0 rg: 80 | total time: 2.63m | eta: 59.5m +step 00074/01500 (4.93%) | loss: 3.285279 | lrm: 1.00 | dt: 2502.72ms | tok/sec: 209,486 | bf16_mfu: 31.98 | epoch: 1 pq: 0 rg: 80 | total time: 2.67m | eta: 59.5m +step 00075/01500 (5.00%) | loss: 3.246838 | lrm: 1.00 | dt: 2496.13ms | tok/sec: 210,040 | bf16_mfu: 32.07 | epoch: 1 pq: 0 rg: 80 | total time: 2.71m | eta: 59.4m +step 00076/01500 (5.07%) | loss: 3.204915 | lrm: 1.00 | dt: 2496.94ms | tok/sec: 209,972 | bf16_mfu: 32.06 | epoch: 1 pq: 0 rg: 80 | total time: 2.75m | eta: 59.4m +step 00077/01500 (5.13%) | loss: 3.173970 | lrm: 1.00 | dt: 2501.91ms | tok/sec: 209,555 | bf16_mfu: 31.99 | epoch: 1 pq: 0 rg: 80 | total time: 2.79m | eta: 59.3m +step 00078/01500 (5.20%) | loss: 3.154224 | lrm: 1.00 | dt: 2505.64ms | tok/sec: 209,243 | bf16_mfu: 31.95 | epoch: 1 pq: 0 rg: 80 | total time: 2.84m | eta: 59.3m +step 00079/01500 (5.27%) | loss: 3.160177 | lrm: 1.00 | dt: 2500.09ms | tok/sec: 209,707 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 0 | total time: 2.88m | eta: 59.2m +step 00080/01500 (5.33%) | loss: 3.208149 | lrm: 1.00 | dt: 2505.87ms | tok/sec: 209,224 | bf16_mfu: 31.94 | epoch: 1 pq: 1 rg: 0 | total time: 2.92m | eta: 59.2m +step 00081/01500 (5.40%) | loss: 3.227993 | lrm: 1.00 | dt: 2497.36ms | tok/sec: 209,936 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 0 | total time: 2.96m | eta: 59.2m +step 00082/01500 (5.47%) | loss: 3.244446 | lrm: 1.00 | dt: 2503.82ms | tok/sec: 209,395 | bf16_mfu: 31.97 | epoch: 1 pq: 1 rg: 0 | total time: 3.00m | eta: 59.1m +step 00083/01500 (5.53%) | loss: 3.228024 | lrm: 1.00 | dt: 2497.13ms | tok/sec: 209,956 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 0 | total time: 3.04m | eta: 59.1m +step 00084/01500 (5.60%) | loss: 3.237289 | lrm: 1.00 | dt: 2495.61ms | tok/sec: 210,084 | bf16_mfu: 32.07 | epoch: 1 pq: 1 rg: 0 | total time: 3.09m | eta: 59.0m +step 00085/01500 (5.67%) | loss: 3.227059 | lrm: 1.00 | dt: 2499.86ms | tok/sec: 209,727 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 0 | total time: 3.13m | eta: 59.0m +step 00086/01500 (5.73%) | loss: 3.235147 | lrm: 1.00 | dt: 2498.19ms | tok/sec: 209,866 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 8 | total time: 3.17m | eta: 58.9m +step 00087/01500 (5.80%) | loss: 3.183798 | lrm: 1.00 | dt: 2497.28ms | tok/sec: 209,943 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 8 | total time: 3.21m | eta: 58.9m +step 00088/01500 (5.87%) | loss: 3.160538 | lrm: 1.00 | dt: 2500.68ms | tok/sec: 209,658 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 8 | total time: 3.25m | eta: 58.9m +step 00089/01500 (5.93%) | loss: 3.142993 | lrm: 1.00 | dt: 2493.69ms | tok/sec: 210,245 | bf16_mfu: 32.10 | epoch: 1 pq: 1 rg: 8 | total time: 3.29m | eta: 58.8m +step 00090/01500 (6.00%) | loss: 3.148296 | lrm: 1.00 | dt: 2500.75ms | tok/sec: 209,652 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 8 | total time: 3.33m | eta: 58.8m +step 00091/01500 (6.07%) | loss: 3.148940 | lrm: 1.00 | dt: 2497.71ms | tok/sec: 209,907 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 8 | total time: 3.38m | eta: 58.7m +step 00092/01500 (6.13%) | loss: 3.138557 | lrm: 1.00 | dt: 2495.00ms | tok/sec: 210,135 | bf16_mfu: 32.08 | epoch: 1 pq: 1 rg: 8 | total time: 3.42m | eta: 58.7m +step 00093/01500 (6.20%) | loss: 3.047582 | lrm: 1.00 | dt: 2499.25ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 16 | total time: 3.46m | eta: 58.6m +step 00094/01500 (6.27%) | loss: 3.023392 | lrm: 1.00 | dt: 2498.33ms | tok/sec: 209,855 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 16 | total time: 3.50m | eta: 58.6m +step 00095/01500 (6.33%) | loss: 2.988202 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 16 | total time: 3.54m | eta: 58.6m +step 00096/01500 (6.40%) | loss: 2.978244 | lrm: 1.00 | dt: 2502.08ms | tok/sec: 209,540 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 16 | total time: 3.58m | eta: 58.5m +step 00097/01500 (6.47%) | loss: 2.943740 | lrm: 1.00 | dt: 2498.33ms | tok/sec: 209,855 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 16 | total time: 3.63m | eta: 58.5m +step 00098/01500 (6.53%) | loss: 2.956456 | lrm: 1.00 | dt: 2501.04ms | tok/sec: 209,628 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 16 | total time: 3.67m | eta: 58.4m +step 00099/01500 (6.60%) | loss: 2.917940 | lrm: 1.00 | dt: 2500.95ms | tok/sec: 209,635 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 16 | total time: 3.71m | eta: 58.4m +Step 00100 | Validation bpb: 0.877004 +Step 00100 | ICL early: 3.2338 late: 3.3452 score: +0.1114 | Induction loss: 13.4303 acc: 0.0038 (random-half: 14.8965) +step 00100/01500 (6.67%) | loss: 2.915524 | lrm: 1.00 | dt: 2497.35ms | tok/sec: 209,937 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 16 | total time: 3.75m | eta: 58.4m +step 00101/01500 (6.73%) | loss: 2.898875 | lrm: 1.00 | dt: 2496.59ms | tok/sec: 210,001 | bf16_mfu: 32.06 | epoch: 1 pq: 1 rg: 24 | total time: 3.79m | eta: 58.3m +step 00102/01500 (6.80%) | loss: 2.925061 | lrm: 1.00 | dt: 2505.95ms | tok/sec: 209,217 | bf16_mfu: 31.94 | epoch: 1 pq: 1 rg: 24 | total time: 3.83m | eta: 58.3m +step 00103/01500 (6.87%) | loss: 2.932929 | lrm: 1.00 | dt: 2496.37ms | tok/sec: 210,020 | bf16_mfu: 32.06 | epoch: 1 pq: 1 rg: 24 | total time: 3.88m | eta: 58.2m +step 00104/01500 (6.93%) | loss: 2.906146 | lrm: 1.00 | dt: 2497.89ms | tok/sec: 209,892 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 24 | total time: 3.92m | eta: 58.2m +step 00105/01500 (7.00%) | loss: 2.907918 | lrm: 1.00 | dt: 2502.32ms | tok/sec: 209,520 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 24 | total time: 3.96m | eta: 58.1m +step 00106/01500 (7.07%) | loss: 2.924005 | lrm: 1.00 | dt: 2506.34ms | tok/sec: 209,184 | bf16_mfu: 31.94 | epoch: 1 pq: 1 rg: 24 | total time: 4.00m | eta: 58.1m +step 00107/01500 (7.13%) | loss: 2.904355 | lrm: 1.00 | dt: 2505.20ms | tok/sec: 209,279 | bf16_mfu: 31.95 | epoch: 1 pq: 1 rg: 24 | total time: 4.04m | eta: 58.1m +step 00108/01500 (7.20%) | loss: 2.956387 | lrm: 1.00 | dt: 2500.24ms | tok/sec: 209,695 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 32 | total time: 4.08m | eta: 58.0m +step 00109/01500 (7.27%) | loss: 2.921634 | lrm: 1.00 | dt: 2498.37ms | tok/sec: 209,852 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 32 | total time: 4.13m | eta: 58.0m +step 00110/01500 (7.33%) | loss: 2.921770 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 32 | total time: 4.17m | eta: 57.9m +step 00111/01500 (7.40%) | loss: 2.914309 | lrm: 1.00 | dt: 2499.01ms | tok/sec: 209,798 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 32 | total time: 4.21m | eta: 57.9m +step 00112/01500 (7.47%) | loss: 2.934415 | lrm: 1.00 | dt: 2499.80ms | tok/sec: 209,731 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 32 | total time: 4.25m | eta: 57.9m +step 00113/01500 (7.53%) | loss: 2.912347 | lrm: 1.00 | dt: 2499.10ms | tok/sec: 209,790 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 32 | total time: 4.29m | eta: 57.8m +step 00114/01500 (7.60%) | loss: 2.897675 | lrm: 1.00 | dt: 2498.62ms | tok/sec: 209,830 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 32 | total time: 4.33m | eta: 57.8m +step 00115/01500 (7.67%) | loss: 2.918555 | lrm: 1.00 | dt: 2496.98ms | tok/sec: 209,968 | bf16_mfu: 32.06 | epoch: 1 pq: 1 rg: 32 | total time: 4.38m | eta: 57.7m +step 00116/01500 (7.73%) | loss: 2.921530 | lrm: 1.00 | dt: 2502.70ms | tok/sec: 209,489 | bf16_mfu: 31.98 | epoch: 1 pq: 1 rg: 40 | total time: 4.42m | eta: 57.7m +step 00117/01500 (7.80%) | loss: 2.926781 | lrm: 1.00 | dt: 2501.53ms | tok/sec: 209,586 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 40 | total time: 4.46m | eta: 57.6m +step 00118/01500 (7.87%) | loss: 2.944181 | lrm: 1.00 | dt: 2503.88ms | tok/sec: 209,390 | bf16_mfu: 31.97 | epoch: 1 pq: 1 rg: 40 | total time: 4.50m | eta: 57.6m +step 00119/01500 (7.93%) | loss: 2.963816 | lrm: 1.00 | dt: 2494.88ms | tok/sec: 210,145 | bf16_mfu: 32.08 | epoch: 1 pq: 1 rg: 40 | total time: 4.54m | eta: 57.6m +step 00120/01500 (8.00%) | loss: 2.978842 | lrm: 1.00 | dt: 2499.18ms | tok/sec: 209,784 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 40 | total time: 4.58m | eta: 57.5m +step 00121/01500 (8.07%) | loss: 2.982057 | lrm: 1.00 | dt: 2500.73ms | tok/sec: 209,653 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 40 | total time: 4.63m | eta: 57.5m +step 00122/01500 (8.13%) | loss: 2.973189 | lrm: 1.00 | dt: 2500.84ms | tok/sec: 209,644 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 40 | total time: 4.67m | eta: 57.4m +step 00123/01500 (8.20%) | loss: 2.962652 | lrm: 1.00 | dt: 2499.48ms | tok/sec: 209,758 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 48 | total time: 4.71m | eta: 57.4m +step 00124/01500 (8.27%) | loss: 2.924283 | lrm: 1.00 | dt: 2503.58ms | tok/sec: 209,415 | bf16_mfu: 31.97 | epoch: 1 pq: 1 rg: 48 | total time: 4.75m | eta: 57.4m +step 00125/01500 (8.33%) | loss: 2.920081 | lrm: 1.00 | dt: 2501.28ms | tok/sec: 209,607 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 48 | total time: 4.79m | eta: 57.3m +step 00126/01500 (8.40%) | loss: 2.920181 | lrm: 1.00 | dt: 2500.93ms | tok/sec: 209,637 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 48 | total time: 4.83m | eta: 57.3m +step 00127/01500 (8.47%) | loss: 2.946619 | lrm: 1.00 | dt: 2500.60ms | tok/sec: 209,665 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 48 | total time: 4.88m | eta: 57.2m +step 00128/01500 (8.53%) | loss: 2.953652 | lrm: 1.00 | dt: 2501.46ms | tok/sec: 209,592 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 48 | total time: 4.92m | eta: 57.2m +step 00129/01500 (8.60%) | loss: 2.943835 | lrm: 1.00 | dt: 2497.84ms | tok/sec: 209,896 | bf16_mfu: 32.05 | epoch: 1 pq: 1 rg: 48 | total time: 4.96m | eta: 57.1m +step 00130/01500 (8.67%) | loss: 2.915199 | lrm: 1.00 | dt: 2508.27ms | tok/sec: 209,023 | bf16_mfu: 31.91 | epoch: 1 pq: 1 rg: 56 | total time: 5.00m | eta: 57.1m +step 00131/01500 (8.73%) | loss: 2.835930 | lrm: 1.00 | dt: 2498.84ms | tok/sec: 209,812 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 56 | total time: 5.04m | eta: 57.1m +step 00132/01500 (8.80%) | loss: 2.796642 | lrm: 1.00 | dt: 2500.51ms | tok/sec: 209,672 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 56 | total time: 5.08m | eta: 57.0m +step 00133/01500 (8.87%) | loss: 2.824124 | lrm: 1.00 | dt: 2496.34ms | tok/sec: 210,022 | bf16_mfu: 32.07 | epoch: 1 pq: 1 rg: 56 | total time: 5.13m | eta: 57.0m +step 00134/01500 (8.93%) | loss: 2.820834 | lrm: 1.00 | dt: 2502.11ms | tok/sec: 209,538 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 56 | total time: 5.17m | eta: 56.9m +step 00135/01500 (9.00%) | loss: 2.827388 | lrm: 1.00 | dt: 2504.91ms | tok/sec: 209,304 | bf16_mfu: 31.96 | epoch: 1 pq: 1 rg: 56 | total time: 5.21m | eta: 56.9m +step 00136/01500 (9.07%) | loss: 2.829499 | lrm: 1.00 | dt: 2501.80ms | tok/sec: 209,564 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 56 | total time: 5.25m | eta: 56.9m +step 00137/01500 (9.13%) | loss: 2.877256 | lrm: 1.00 | dt: 2499.21ms | tok/sec: 209,781 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 56 | total time: 5.29m | eta: 56.8m +step 00138/01500 (9.20%) | loss: 2.871995 | lrm: 1.00 | dt: 2499.98ms | tok/sec: 209,717 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 64 | total time: 5.34m | eta: 56.8m +step 00139/01500 (9.27%) | loss: 2.857608 | lrm: 1.00 | dt: 2498.79ms | tok/sec: 209,817 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 64 | total time: 5.38m | eta: 56.7m +step 00140/01500 (9.33%) | loss: 2.870178 | lrm: 1.00 | dt: 2502.12ms | tok/sec: 209,537 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 64 | total time: 5.42m | eta: 56.7m +step 00141/01500 (9.40%) | loss: 2.891083 | lrm: 1.00 | dt: 2502.49ms | tok/sec: 209,506 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 64 | total time: 5.46m | eta: 56.6m +step 00142/01500 (9.47%) | loss: 2.938610 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 64 | total time: 5.50m | eta: 56.6m +step 00143/01500 (9.53%) | loss: 2.966213 | lrm: 1.00 | dt: 2498.90ms | tok/sec: 209,807 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 64 | total time: 5.54m | eta: 56.6m +step 00144/01500 (9.60%) | loss: 2.975140 | lrm: 1.00 | dt: 2498.56ms | tok/sec: 209,835 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 64 | total time: 5.59m | eta: 56.5m +step 00145/01500 (9.67%) | loss: 2.964519 | lrm: 1.00 | dt: 2749.59ms | tok/sec: 190,678 | bf16_mfu: 29.11 | epoch: 1 pq: 1 rg: 72 | total time: 5.63m | eta: 56.5m +step 00146/01500 (9.73%) | loss: 2.953216 | lrm: 1.00 | dt: 2499.90ms | tok/sec: 209,723 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 72 | total time: 5.67m | eta: 56.5m +step 00147/01500 (9.80%) | loss: 2.914044 | lrm: 1.00 | dt: 2500.78ms | tok/sec: 209,649 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 72 | total time: 5.71m | eta: 56.4m +step 00148/01500 (9.87%) | loss: 2.920703 | lrm: 1.00 | dt: 2499.11ms | tok/sec: 209,789 | bf16_mfu: 32.03 | epoch: 1 pq: 1 rg: 72 | total time: 5.76m | eta: 56.4m +step 00149/01500 (9.93%) | loss: 2.899524 | lrm: 1.00 | dt: 2498.52ms | tok/sec: 209,839 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 72 | total time: 5.80m | eta: 56.3m +step 00150/01500 (10.00%) | loss: 2.917274 | lrm: 1.00 | dt: 2502.08ms | tok/sec: 209,540 | bf16_mfu: 31.99 | epoch: 1 pq: 1 rg: 72 | total time: 5.84m | eta: 56.3m +step 00151/01500 (10.07%) | loss: 2.939549 | lrm: 1.00 | dt: 2498.40ms | tok/sec: 209,849 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 72 | total time: 5.88m | eta: 56.3m +step 00152/01500 (10.13%) | loss: 2.917839 | lrm: 1.00 | dt: 2499.72ms | tok/sec: 209,738 | bf16_mfu: 32.02 | epoch: 1 pq: 1 rg: 80 | total time: 5.92m | eta: 56.2m +step 00153/01500 (10.20%) | loss: 2.856035 | lrm: 1.00 | dt: 2500.77ms | tok/sec: 209,650 | bf16_mfu: 32.01 | epoch: 1 pq: 1 rg: 80 | total time: 5.96m | eta: 56.2m +step 00154/01500 (10.27%) | loss: 2.805288 | lrm: 1.00 | dt: 2503.66ms | tok/sec: 209,408 | bf16_mfu: 31.97 | epoch: 1 pq: 1 rg: 80 | total time: 6.01m | eta: 56.1m +step 00155/01500 (10.33%) | loss: 2.785939 | lrm: 1.00 | dt: 2501.78ms | tok/sec: 209,566 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 80 | total time: 6.05m | eta: 56.1m +step 00156/01500 (10.40%) | loss: 2.786126 | lrm: 1.00 | dt: 2501.44ms | tok/sec: 209,594 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 80 | total time: 6.09m | eta: 56.1m +step 00157/01500 (10.47%) | loss: 2.739186 | lrm: 1.00 | dt: 2501.45ms | tok/sec: 209,593 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 80 | total time: 6.13m | eta: 56.0m +step 00158/01500 (10.53%) | loss: 2.744949 | lrm: 1.00 | dt: 2501.34ms | tok/sec: 209,603 | bf16_mfu: 32.00 | epoch: 1 pq: 1 rg: 80 | total time: 6.17m | eta: 56.0m +step 00159/01500 (10.60%) | loss: 2.774082 | lrm: 1.00 | dt: 2498.28ms | tok/sec: 209,859 | bf16_mfu: 32.04 | epoch: 1 pq: 1 rg: 80 | total time: 6.21m | eta: 55.9m +step 00160/01500 (10.67%) | loss: 2.714753 | lrm: 1.00 | dt: 2510.83ms | tok/sec: 208,810 | bf16_mfu: 31.88 | epoch: 1 pq: 2 rg: 0 | total time: 6.26m | eta: 55.9m +step 00161/01500 (10.73%) | loss: 2.688827 | lrm: 1.00 | dt: 2498.04ms | tok/sec: 209,879 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 0 | total time: 6.30m | eta: 55.8m +step 00162/01500 (10.80%) | loss: 2.656541 | lrm: 1.00 | dt: 2501.99ms | tok/sec: 209,548 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 0 | total time: 6.34m | eta: 55.8m +step 00163/01500 (10.87%) | loss: 2.671044 | lrm: 1.00 | dt: 2506.87ms | tok/sec: 209,140 | bf16_mfu: 31.93 | epoch: 1 pq: 2 rg: 0 | total time: 6.38m | eta: 55.8m +step 00164/01500 (10.93%) | loss: 2.712461 | lrm: 1.00 | dt: 2503.64ms | tok/sec: 209,410 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 0 | total time: 6.42m | eta: 55.7m +step 00165/01500 (11.00%) | loss: 2.730720 | lrm: 1.00 | dt: 2504.83ms | tok/sec: 209,310 | bf16_mfu: 31.96 | epoch: 1 pq: 2 rg: 0 | total time: 6.46m | eta: 55.7m +step 00166/01500 (11.07%) | loss: 2.751532 | lrm: 1.00 | dt: 2500.21ms | tok/sec: 209,697 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 0 | total time: 6.51m | eta: 55.6m +step 00167/01500 (11.13%) | loss: 2.798152 | lrm: 1.00 | dt: 2501.45ms | tok/sec: 209,593 | bf16_mfu: 32.00 | epoch: 1 pq: 2 rg: 8 | total time: 6.55m | eta: 55.6m +step 00168/01500 (11.20%) | loss: 2.865066 | lrm: 1.00 | dt: 2502.38ms | tok/sec: 209,515 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 8 | total time: 6.59m | eta: 55.6m +step 00169/01500 (11.27%) | loss: 2.860980 | lrm: 1.00 | dt: 2504.08ms | tok/sec: 209,373 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 8 | total time: 6.63m | eta: 55.5m +step 00170/01500 (11.33%) | loss: 2.853965 | lrm: 1.00 | dt: 2497.51ms | tok/sec: 209,924 | bf16_mfu: 32.05 | epoch: 1 pq: 2 rg: 8 | total time: 6.67m | eta: 55.5m +step 00171/01500 (11.40%) | loss: 2.882350 | lrm: 1.00 | dt: 2503.40ms | tok/sec: 209,430 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 8 | total time: 6.71m | eta: 55.4m +step 00172/01500 (11.47%) | loss: 2.881416 | lrm: 1.00 | dt: 2501.18ms | tok/sec: 209,616 | bf16_mfu: 32.00 | epoch: 1 pq: 2 rg: 8 | total time: 6.76m | eta: 55.4m +step 00173/01500 (11.53%) | loss: 2.835252 | lrm: 1.00 | dt: 2500.53ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 8 | total time: 6.80m | eta: 55.3m +step 00174/01500 (11.60%) | loss: 2.866169 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 8 | total time: 6.84m | eta: 55.3m +step 00175/01500 (11.67%) | loss: 2.884370 | lrm: 1.00 | dt: 2500.46ms | tok/sec: 209,676 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 16 | total time: 6.88m | eta: 55.3m +step 00176/01500 (11.73%) | loss: 2.859987 | lrm: 1.00 | dt: 2504.46ms | tok/sec: 209,341 | bf16_mfu: 31.96 | epoch: 1 pq: 2 rg: 16 | total time: 6.92m | eta: 55.2m +step 00177/01500 (11.80%) | loss: 2.856667 | lrm: 1.00 | dt: 2503.79ms | tok/sec: 209,397 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 16 | total time: 6.97m | eta: 55.2m +step 00178/01500 (11.87%) | loss: 2.856164 | lrm: 1.00 | dt: 2504.08ms | tok/sec: 209,373 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 16 | total time: 7.01m | eta: 55.1m +step 00179/01500 (11.93%) | loss: 2.863707 | lrm: 1.00 | dt: 2501.25ms | tok/sec: 209,610 | bf16_mfu: 32.00 | epoch: 1 pq: 2 rg: 16 | total time: 7.05m | eta: 55.1m +step 00180/01500 (12.00%) | loss: 2.833860 | lrm: 1.00 | dt: 2499.77ms | tok/sec: 209,734 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 16 | total time: 7.09m | eta: 55.1m +step 00181/01500 (12.07%) | loss: 2.809028 | lrm: 1.00 | dt: 2500.77ms | tok/sec: 209,650 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 16 | total time: 7.13m | eta: 55.0m +step 00182/01500 (12.13%) | loss: 2.818415 | lrm: 1.00 | dt: 2499.14ms | tok/sec: 209,787 | bf16_mfu: 32.03 | epoch: 1 pq: 2 rg: 24 | total time: 7.17m | eta: 55.0m +step 00183/01500 (12.20%) | loss: 2.826522 | lrm: 1.00 | dt: 2502.62ms | tok/sec: 209,495 | bf16_mfu: 31.98 | epoch: 1 pq: 2 rg: 24 | total time: 7.22m | eta: 54.9m +step 00184/01500 (12.27%) | loss: 2.807088 | lrm: 1.00 | dt: 2502.90ms | tok/sec: 209,472 | bf16_mfu: 31.98 | epoch: 1 pq: 2 rg: 24 | total time: 7.26m | eta: 54.9m +step 00185/01500 (12.33%) | loss: 2.785449 | lrm: 1.00 | dt: 2498.21ms | tok/sec: 209,865 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 24 | total time: 7.30m | eta: 54.8m +step 00186/01500 (12.40%) | loss: 2.777982 | lrm: 1.00 | dt: 2502.86ms | tok/sec: 209,475 | bf16_mfu: 31.98 | epoch: 1 pq: 2 rg: 24 | total time: 7.34m | eta: 54.8m +step 00187/01500 (12.47%) | loss: 2.793644 | lrm: 1.00 | dt: 2499.91ms | tok/sec: 209,722 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 24 | total time: 7.38m | eta: 54.8m +step 00188/01500 (12.53%) | loss: 2.772648 | lrm: 1.00 | dt: 2500.69ms | tok/sec: 209,656 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 24 | total time: 7.42m | eta: 54.7m +step 00189/01500 (12.60%) | loss: 2.773812 | lrm: 1.00 | dt: 2505.01ms | tok/sec: 209,295 | bf16_mfu: 31.95 | epoch: 1 pq: 2 rg: 32 | total time: 7.47m | eta: 54.7m +step 00190/01500 (12.67%) | loss: 2.774943 | lrm: 1.00 | dt: 2503.36ms | tok/sec: 209,433 | bf16_mfu: 31.98 | epoch: 1 pq: 2 rg: 32 | total time: 7.51m | eta: 54.6m +step 00191/01500 (12.73%) | loss: 2.763321 | lrm: 1.00 | dt: 2500.16ms | tok/sec: 209,701 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 32 | total time: 7.55m | eta: 54.6m +step 00192/01500 (12.80%) | loss: 2.728710 | lrm: 1.00 | dt: 2502.47ms | tok/sec: 209,508 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 32 | total time: 7.59m | eta: 54.6m +step 00193/01500 (12.87%) | loss: 2.780058 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 32 | total time: 7.63m | eta: 54.5m +step 00194/01500 (12.93%) | loss: 2.777232 | lrm: 1.00 | dt: 2500.12ms | tok/sec: 209,705 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 32 | total time: 7.67m | eta: 54.5m +step 00195/01500 (13.00%) | loss: 2.808145 | lrm: 1.00 | dt: 2500.63ms | tok/sec: 209,662 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 32 | total time: 7.72m | eta: 54.4m +step 00196/01500 (13.07%) | loss: 2.792706 | lrm: 1.00 | dt: 2497.46ms | tok/sec: 209,928 | bf16_mfu: 32.05 | epoch: 1 pq: 2 rg: 32 | total time: 7.76m | eta: 54.4m +step 00197/01500 (13.13%) | loss: 2.789577 | lrm: 1.00 | dt: 2500.91ms | tok/sec: 209,638 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 40 | total time: 7.80m | eta: 54.3m +step 00198/01500 (13.20%) | loss: 2.842831 | lrm: 1.00 | dt: 2499.25ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 2 rg: 40 | total time: 7.84m | eta: 54.3m +step 00199/01500 (13.27%) | loss: 2.854176 | lrm: 1.00 | dt: 2496.81ms | tok/sec: 209,983 | bf16_mfu: 32.06 | epoch: 1 pq: 2 rg: 40 | total time: 7.88m | eta: 54.3m +Step 00200 | Validation bpb: 0.809731 +Step 00200 | ICL early: 3.0344 late: 3.0982 score: +0.0638 | Induction loss: 12.2065 acc: 0.0161 (random-half: 14.5413) +step 00200/01500 (13.33%) | loss: 2.840186 | lrm: 1.00 | dt: 2499.73ms | tok/sec: 209,738 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 40 | total time: 7.92m | eta: 54.2m +step 00201/01500 (13.40%) | loss: 2.777129 | lrm: 1.00 | dt: 2497.93ms | tok/sec: 209,888 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 40 | total time: 7.97m | eta: 54.2m +step 00202/01500 (13.47%) | loss: 2.757498 | lrm: 1.00 | dt: 2500.63ms | tok/sec: 209,662 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 40 | total time: 8.01m | eta: 54.1m +step 00203/01500 (13.53%) | loss: 2.731559 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 2 rg: 40 | total time: 8.05m | eta: 54.1m +step 00204/01500 (13.60%) | loss: 2.736987 | lrm: 1.00 | dt: 2499.95ms | tok/sec: 209,719 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 48 | total time: 8.09m | eta: 54.0m +step 00205/01500 (13.67%) | loss: 2.719595 | lrm: 1.00 | dt: 2495.74ms | tok/sec: 210,073 | bf16_mfu: 32.07 | epoch: 1 pq: 2 rg: 48 | total time: 8.13m | eta: 54.0m +step 00206/01500 (13.73%) | loss: 2.702856 | lrm: 1.00 | dt: 2503.57ms | tok/sec: 209,416 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 48 | total time: 8.17m | eta: 54.0m +step 00207/01500 (13.80%) | loss: 2.743100 | lrm: 1.00 | dt: 2500.33ms | tok/sec: 209,687 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 48 | total time: 8.22m | eta: 53.9m +step 00208/01500 (13.87%) | loss: 2.731707 | lrm: 1.00 | dt: 2498.31ms | tok/sec: 209,857 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 48 | total time: 8.26m | eta: 53.9m +step 00209/01500 (13.93%) | loss: 2.750448 | lrm: 1.00 | dt: 2505.58ms | tok/sec: 209,248 | bf16_mfu: 31.95 | epoch: 1 pq: 2 rg: 48 | total time: 8.30m | eta: 53.8m +step 00210/01500 (14.00%) | loss: 2.651502 | lrm: 1.00 | dt: 2498.41ms | tok/sec: 209,848 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 48 | total time: 8.34m | eta: 53.8m +step 00211/01500 (14.07%) | loss: 2.643204 | lrm: 1.00 | dt: 2507.21ms | tok/sec: 209,112 | bf16_mfu: 31.93 | epoch: 1 pq: 2 rg: 56 | total time: 8.38m | eta: 53.8m +step 00212/01500 (14.13%) | loss: 2.636580 | lrm: 1.00 | dt: 2502.34ms | tok/sec: 209,518 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 56 | total time: 8.42m | eta: 53.7m +step 00213/01500 (14.20%) | loss: 2.646286 | lrm: 1.00 | dt: 2501.99ms | tok/sec: 209,548 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 56 | total time: 8.47m | eta: 53.7m +step 00214/01500 (14.27%) | loss: 2.657232 | lrm: 1.00 | dt: 2501.48ms | tok/sec: 209,591 | bf16_mfu: 32.00 | epoch: 1 pq: 2 rg: 56 | total time: 8.51m | eta: 53.6m +step 00215/01500 (14.33%) | loss: 2.670429 | lrm: 1.00 | dt: 2499.40ms | tok/sec: 209,765 | bf16_mfu: 32.03 | epoch: 1 pq: 2 rg: 56 | total time: 8.55m | eta: 53.6m +step 00216/01500 (14.40%) | loss: 2.685346 | lrm: 1.00 | dt: 2502.41ms | tok/sec: 209,513 | bf16_mfu: 31.99 | epoch: 1 pq: 2 rg: 56 | total time: 8.59m | eta: 53.5m +step 00217/01500 (14.47%) | loss: 2.687898 | lrm: 1.00 | dt: 2503.66ms | tok/sec: 209,408 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 56 | total time: 8.63m | eta: 53.5m +step 00218/01500 (14.53%) | loss: 2.686027 | lrm: 1.00 | dt: 2499.84ms | tok/sec: 209,728 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 64 | total time: 8.67m | eta: 53.5m +step 00219/01500 (14.60%) | loss: 2.746372 | lrm: 1.00 | dt: 2905.22ms | tok/sec: 180,464 | bf16_mfu: 27.55 | epoch: 1 pq: 2 rg: 64 | total time: 8.72m | eta: 53.5m +step 00220/01500 (14.67%) | loss: 2.746497 | lrm: 1.00 | dt: 2499.56ms | tok/sec: 209,752 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 64 | total time: 8.76m | eta: 53.4m +step 00221/01500 (14.73%) | loss: 2.755573 | lrm: 1.00 | dt: 2504.56ms | tok/sec: 209,333 | bf16_mfu: 31.96 | epoch: 1 pq: 2 rg: 64 | total time: 8.81m | eta: 53.4m +step 00222/01500 (14.80%) | loss: 2.775914 | lrm: 1.00 | dt: 2496.10ms | tok/sec: 210,043 | bf16_mfu: 32.07 | epoch: 1 pq: 2 rg: 64 | total time: 8.85m | eta: 53.3m +step 00223/01500 (14.87%) | loss: 2.712128 | lrm: 1.00 | dt: 2497.29ms | tok/sec: 209,942 | bf16_mfu: 32.05 | epoch: 1 pq: 2 rg: 64 | total time: 8.89m | eta: 53.3m +step 00224/01500 (14.93%) | loss: 2.699746 | lrm: 1.00 | dt: 2504.78ms | tok/sec: 209,314 | bf16_mfu: 31.96 | epoch: 1 pq: 2 rg: 64 | total time: 8.93m | eta: 53.3m +step 00225/01500 (15.00%) | loss: 2.695812 | lrm: 1.00 | dt: 2500.45ms | tok/sec: 209,677 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 64 | total time: 8.97m | eta: 53.2m +step 00226/01500 (15.07%) | loss: 2.651124 | lrm: 1.00 | dt: 2501.00ms | tok/sec: 209,631 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 72 | total time: 9.01m | eta: 53.2m +step 00227/01500 (15.13%) | loss: 2.630231 | lrm: 1.00 | dt: 2496.61ms | tok/sec: 210,000 | bf16_mfu: 32.06 | epoch: 1 pq: 2 rg: 72 | total time: 9.06m | eta: 53.1m +step 00228/01500 (15.20%) | loss: 2.631078 | lrm: 1.00 | dt: 2503.45ms | tok/sec: 209,425 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 72 | total time: 9.10m | eta: 53.1m +step 00229/01500 (15.27%) | loss: 2.615961 | lrm: 1.00 | dt: 2503.66ms | tok/sec: 209,408 | bf16_mfu: 31.97 | epoch: 1 pq: 2 rg: 72 | total time: 9.14m | eta: 53.0m +step 00230/01500 (15.33%) | loss: 2.634345 | lrm: 1.00 | dt: 2494.33ms | tok/sec: 210,191 | bf16_mfu: 32.09 | epoch: 1 pq: 2 rg: 72 | total time: 9.18m | eta: 53.0m +step 00231/01500 (15.40%) | loss: 2.658159 | lrm: 1.00 | dt: 2498.41ms | tok/sec: 209,848 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 72 | total time: 9.22m | eta: 53.0m +step 00232/01500 (15.47%) | loss: 2.665212 | lrm: 1.00 | dt: 2499.67ms | tok/sec: 209,743 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 72 | total time: 9.26m | eta: 52.9m +step 00233/01500 (15.53%) | loss: 2.702525 | lrm: 1.00 | dt: 2499.89ms | tok/sec: 209,724 | bf16_mfu: 32.02 | epoch: 1 pq: 2 rg: 72 | total time: 9.31m | eta: 52.9m +step 00234/01500 (15.60%) | loss: 2.688055 | lrm: 1.00 | dt: 2498.00ms | tok/sec: 209,883 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 80 | total time: 9.35m | eta: 52.8m +step 00235/01500 (15.67%) | loss: 2.677379 | lrm: 1.00 | dt: 2500.49ms | tok/sec: 209,674 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 80 | total time: 9.39m | eta: 52.8m +step 00236/01500 (15.73%) | loss: 2.665520 | lrm: 1.00 | dt: 2499.24ms | tok/sec: 209,778 | bf16_mfu: 32.03 | epoch: 1 pq: 2 rg: 80 | total time: 9.43m | eta: 52.7m +step 00237/01500 (15.80%) | loss: 2.635362 | lrm: 1.00 | dt: 2498.00ms | tok/sec: 209,883 | bf16_mfu: 32.04 | epoch: 1 pq: 2 rg: 80 | total time: 9.47m | eta: 52.7m +step 00238/01500 (15.87%) | loss: 2.681989 | lrm: 1.00 | dt: 2500.71ms | tok/sec: 209,656 | bf16_mfu: 32.01 | epoch: 1 pq: 2 rg: 80 | total time: 9.51m | eta: 52.7m +step 00239/01500 (15.93%) | loss: 2.665659 | lrm: 1.00 | dt: 2497.48ms | tok/sec: 209,926 | bf16_mfu: 32.05 | epoch: 1 pq: 2 rg: 80 | total time: 9.56m | eta: 52.6m +step 00240/01500 (16.00%) | loss: 2.653508 | lrm: 1.00 | dt: 2498.78ms | tok/sec: 209,817 | bf16_mfu: 32.03 | epoch: 1 pq: 2 rg: 80 | total time: 9.60m | eta: 52.6m +step 00241/01500 (16.07%) | loss: 2.631025 | lrm: 1.00 | dt: 2498.98ms | tok/sec: 209,800 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 0 | total time: 9.64m | eta: 52.5m +step 00242/01500 (16.13%) | loss: 2.629892 | lrm: 1.00 | dt: 2498.83ms | tok/sec: 209,813 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 0 | total time: 9.68m | eta: 52.5m +step 00243/01500 (16.20%) | loss: 2.642400 | lrm: 1.00 | dt: 2498.98ms | tok/sec: 209,800 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 0 | total time: 9.72m | eta: 52.5m +step 00244/01500 (16.27%) | loss: 2.664730 | lrm: 1.00 | dt: 2497.75ms | tok/sec: 209,903 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 0 | total time: 9.76m | eta: 52.4m +step 00245/01500 (16.33%) | loss: 2.677625 | lrm: 1.00 | dt: 2496.70ms | tok/sec: 209,992 | bf16_mfu: 32.06 | epoch: 1 pq: 3 rg: 0 | total time: 9.81m | eta: 52.4m +step 00246/01500 (16.40%) | loss: 2.619270 | lrm: 1.00 | dt: 2499.02ms | tok/sec: 209,797 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 0 | total time: 9.85m | eta: 52.3m +step 00247/01500 (16.47%) | loss: 2.646338 | lrm: 1.00 | dt: 2500.25ms | tok/sec: 209,693 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 0 | total time: 9.89m | eta: 52.3m +step 00248/01500 (16.53%) | loss: 2.684204 | lrm: 1.00 | dt: 2505.23ms | tok/sec: 209,277 | bf16_mfu: 31.95 | epoch: 1 pq: 3 rg: 0 | total time: 9.93m | eta: 52.2m +step 00249/01500 (16.60%) | loss: 2.700333 | lrm: 1.00 | dt: 2505.08ms | tok/sec: 209,290 | bf16_mfu: 31.95 | epoch: 1 pq: 3 rg: 8 | total time: 9.97m | eta: 52.2m +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au, and the atomic number of gold is 10. The atomic number of +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today is Saturday, tomorrow will be Sunday. If today is Sunday +<|bos|>The opposite of hot is cold. Hot is a temperature that is above the boiling point of water. Cold +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like blue because it makes me feel happy. I like blue because +<|bos|>If 5*x + 3 = 13, then x is 3. If x = 3, then x = 3. If +2026-08-10 21:35:19,358 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank4.pt +2026-08-10 21:35:37,688 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank3.pt +2026-08-10 21:35:39,857 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank5.pt +2026-08-10 21:35:50,438 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank1.pt +2026-08-10 21:35:58,952 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank7.pt +2026-08-10 21:35:59,184 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank2.pt +2026-08-10 21:35:59,240 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank6.pt +2026-08-10 21:37:00,283 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_000250.pt +2026-08-10 21:37:00,285 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_000250.json +2026-08-10 21:37:07,539 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000250_rank0.pt +step 00250/01500 (16.67%) | loss: 2.701050 | lrm: 1.00 | dt: 2475.05ms | tok/sec: 211,828 | bf16_mfu: 32.34 | epoch: 1 pq: 3 rg: 8 | total time: 10.01m | eta: 52.2m +step 00251/01500 (16.73%) | loss: 2.694390 | lrm: 1.00 | dt: 2482.94ms | tok/sec: 211,156 | bf16_mfu: 32.24 | epoch: 1 pq: 3 rg: 8 | total time: 10.06m | eta: 52.1m +step 00252/01500 (16.80%) | loss: 2.676868 | lrm: 1.00 | dt: 2497.56ms | tok/sec: 209,920 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 8 | total time: 10.10m | eta: 52.1m +step 00253/01500 (16.87%) | loss: 2.672160 | lrm: 1.00 | dt: 2491.61ms | tok/sec: 210,421 | bf16_mfu: 32.13 | epoch: 1 pq: 3 rg: 8 | total time: 10.14m | eta: 52.0m +step 00254/01500 (16.93%) | loss: 2.669649 | lrm: 1.00 | dt: 2496.71ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 3 rg: 8 | total time: 10.18m | eta: 52.0m +step 00255/01500 (17.00%) | loss: 2.682889 | lrm: 1.00 | dt: 2498.70ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 8 | total time: 10.22m | eta: 51.9m +step 00256/01500 (17.07%) | loss: 2.690229 | lrm: 1.00 | dt: 2500.71ms | tok/sec: 209,655 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 8 | total time: 10.26m | eta: 51.9m +step 00257/01500 (17.13%) | loss: 2.713785 | lrm: 1.00 | dt: 2500.17ms | tok/sec: 209,701 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 16 | total time: 10.30m | eta: 51.9m +step 00258/01500 (17.20%) | loss: 2.744359 | lrm: 1.00 | dt: 2495.70ms | tok/sec: 210,076 | bf16_mfu: 32.07 | epoch: 1 pq: 3 rg: 16 | total time: 10.35m | eta: 51.8m +step 00259/01500 (17.27%) | loss: 2.716034 | lrm: 1.00 | dt: 2494.01ms | tok/sec: 210,219 | bf16_mfu: 32.10 | epoch: 1 pq: 3 rg: 16 | total time: 10.39m | eta: 51.8m +step 00260/01500 (17.33%) | loss: 2.699451 | lrm: 1.00 | dt: 2492.83ms | tok/sec: 210,318 | bf16_mfu: 32.11 | epoch: 1 pq: 3 rg: 16 | total time: 10.43m | eta: 51.7m +step 00261/01500 (17.40%) | loss: 2.692689 | lrm: 1.00 | dt: 2498.95ms | tok/sec: 209,803 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 16 | total time: 10.47m | eta: 51.7m +step 00262/01500 (17.47%) | loss: 2.710181 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 16 | total time: 10.51m | eta: 51.6m +step 00263/01500 (17.53%) | loss: 2.706901 | lrm: 1.00 | dt: 2495.49ms | tok/sec: 210,093 | bf16_mfu: 32.08 | epoch: 1 pq: 3 rg: 16 | total time: 10.55m | eta: 51.6m +step 00264/01500 (17.60%) | loss: 2.721221 | lrm: 1.00 | dt: 2496.82ms | tok/sec: 209,981 | bf16_mfu: 32.06 | epoch: 1 pq: 3 rg: 16 | total time: 10.60m | eta: 51.6m +step 00265/01500 (17.67%) | loss: 2.756175 | lrm: 1.00 | dt: 2499.27ms | tok/sec: 209,776 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 24 | total time: 10.64m | eta: 51.5m +step 00266/01500 (17.73%) | loss: 2.752914 | lrm: 1.00 | dt: 2495.08ms | tok/sec: 210,129 | bf16_mfu: 32.08 | epoch: 1 pq: 3 rg: 24 | total time: 10.68m | eta: 51.5m +step 00267/01500 (17.80%) | loss: 2.741282 | lrm: 1.00 | dt: 2497.50ms | tok/sec: 209,925 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 24 | total time: 10.72m | eta: 51.4m +step 00268/01500 (17.87%) | loss: 2.730078 | lrm: 1.00 | dt: 2498.35ms | tok/sec: 209,853 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 24 | total time: 10.76m | eta: 51.4m +step 00269/01500 (17.93%) | loss: 2.656281 | lrm: 1.00 | dt: 2490.78ms | tok/sec: 210,491 | bf16_mfu: 32.14 | epoch: 1 pq: 3 rg: 24 | total time: 10.80m | eta: 51.4m +step 00270/01500 (18.00%) | loss: 2.609942 | lrm: 1.00 | dt: 2491.76ms | tok/sec: 210,408 | bf16_mfu: 32.12 | epoch: 1 pq: 3 rg: 24 | total time: 10.85m | eta: 51.3m +step 00271/01500 (18.07%) | loss: 2.616283 | lrm: 1.00 | dt: 2497.39ms | tok/sec: 209,934 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 24 | total time: 10.89m | eta: 51.3m +step 00272/01500 (18.13%) | loss: 2.665154 | lrm: 1.00 | dt: 2495.60ms | tok/sec: 210,084 | bf16_mfu: 32.07 | epoch: 1 pq: 3 rg: 24 | total time: 10.93m | eta: 51.2m +step 00273/01500 (18.20%) | loss: 2.688141 | lrm: 1.00 | dt: 2504.76ms | tok/sec: 209,316 | bf16_mfu: 31.96 | epoch: 1 pq: 3 rg: 32 | total time: 10.97m | eta: 51.2m +step 00274/01500 (18.27%) | loss: 2.728167 | lrm: 1.00 | dt: 2500.42ms | tok/sec: 209,679 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 32 | total time: 11.01m | eta: 51.1m +step 00275/01500 (18.33%) | loss: 2.761949 | lrm: 1.00 | dt: 2499.35ms | tok/sec: 209,770 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 32 | total time: 11.05m | eta: 51.1m +step 00276/01500 (18.40%) | loss: 2.759468 | lrm: 1.00 | dt: 2501.02ms | tok/sec: 209,629 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 32 | total time: 11.10m | eta: 51.1m +step 00277/01500 (18.47%) | loss: 2.734210 | lrm: 1.00 | dt: 2505.11ms | tok/sec: 209,287 | bf16_mfu: 31.95 | epoch: 1 pq: 3 rg: 32 | total time: 11.14m | eta: 51.0m +step 00278/01500 (18.53%) | loss: 2.710704 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 32 | total time: 11.18m | eta: 51.0m +step 00279/01500 (18.60%) | loss: 2.695947 | lrm: 1.00 | dt: 2500.23ms | tok/sec: 209,695 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 32 | total time: 11.22m | eta: 50.9m +step 00280/01500 (18.67%) | loss: 2.661022 | lrm: 1.00 | dt: 2502.46ms | tok/sec: 209,509 | bf16_mfu: 31.99 | epoch: 1 pq: 3 rg: 32 | total time: 11.26m | eta: 50.9m +step 00281/01500 (18.73%) | loss: 2.661178 | lrm: 1.00 | dt: 2500.31ms | tok/sec: 209,689 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 40 | total time: 11.30m | eta: 50.8m +step 00282/01500 (18.80%) | loss: 2.656587 | lrm: 1.00 | dt: 2496.01ms | tok/sec: 210,050 | bf16_mfu: 32.07 | epoch: 1 pq: 3 rg: 40 | total time: 11.35m | eta: 50.8m +step 00283/01500 (18.87%) | loss: 2.667393 | lrm: 1.00 | dt: 2497.85ms | tok/sec: 209,895 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 40 | total time: 11.39m | eta: 50.8m +step 00284/01500 (18.93%) | loss: 2.645016 | lrm: 1.00 | dt: 2499.81ms | tok/sec: 209,731 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 40 | total time: 11.43m | eta: 50.7m +step 00285/01500 (19.00%) | loss: 2.658358 | lrm: 1.00 | dt: 2497.54ms | tok/sec: 209,921 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 40 | total time: 11.47m | eta: 50.7m +step 00286/01500 (19.07%) | loss: 2.648874 | lrm: 1.00 | dt: 2499.08ms | tok/sec: 209,792 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 40 | total time: 11.51m | eta: 50.6m +step 00287/01500 (19.13%) | loss: 2.643574 | lrm: 1.00 | dt: 2501.42ms | tok/sec: 209,595 | bf16_mfu: 32.00 | epoch: 1 pq: 3 rg: 40 | total time: 11.55m | eta: 50.6m +step 00288/01500 (19.20%) | loss: 2.632357 | lrm: 1.00 | dt: 2498.17ms | tok/sec: 209,868 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 40 | total time: 11.60m | eta: 50.6m +step 00289/01500 (19.27%) | loss: 2.605199 | lrm: 1.00 | dt: 2500.52ms | tok/sec: 209,671 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 48 | total time: 11.64m | eta: 50.5m +step 00290/01500 (19.33%) | loss: 2.599687 | lrm: 1.00 | dt: 2500.76ms | tok/sec: 209,651 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 48 | total time: 11.68m | eta: 50.5m +step 00291/01500 (19.40%) | loss: 2.565944 | lrm: 1.00 | dt: 2498.25ms | tok/sec: 209,862 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 48 | total time: 11.72m | eta: 50.4m +step 00292/01500 (19.47%) | loss: 2.578022 | lrm: 1.00 | dt: 2498.69ms | tok/sec: 209,825 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 48 | total time: 11.76m | eta: 50.4m +step 00293/01500 (19.53%) | loss: 2.594464 | lrm: 1.00 | dt: 2498.45ms | tok/sec: 209,845 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 48 | total time: 11.80m | eta: 50.3m +step 00294/01500 (19.60%) | loss: 2.617919 | lrm: 1.00 | dt: 2497.29ms | tok/sec: 209,943 | bf16_mfu: 32.05 | epoch: 1 pq: 3 rg: 48 | total time: 11.85m | eta: 50.3m +step 00295/01500 (19.67%) | loss: 2.630398 | lrm: 1.00 | dt: 2500.08ms | tok/sec: 209,708 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 48 | total time: 11.89m | eta: 50.3m +step 00296/01500 (19.73%) | loss: 2.588017 | lrm: 1.00 | dt: 2498.17ms | tok/sec: 209,868 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 48 | total time: 11.93m | eta: 50.2m +step 00297/01500 (19.80%) | loss: 2.613355 | lrm: 1.00 | dt: 2498.17ms | tok/sec: 209,868 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 56 | total time: 11.97m | eta: 50.2m +step 00298/01500 (19.87%) | loss: 2.601538 | lrm: 1.00 | dt: 2764.71ms | tok/sec: 189,635 | bf16_mfu: 28.95 | epoch: 1 pq: 3 rg: 56 | total time: 12.02m | eta: 50.2m +step 00299/01500 (19.93%) | loss: 2.582510 | lrm: 1.00 | dt: 2498.32ms | tok/sec: 209,855 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 56 | total time: 12.06m | eta: 50.1m +Step 00300 | Validation bpb: 0.784707 +Step 00300 | ICL early: 2.9330 late: 3.0051 score: +0.0720 | Induction loss: 11.5097 acc: 0.0289 (random-half: 14.3222) +step 00300/01500 (20.00%) | loss: 2.581956 | lrm: 1.00 | dt: 2493.35ms | tok/sec: 210,274 | bf16_mfu: 32.10 | epoch: 1 pq: 3 rg: 56 | total time: 12.10m | eta: 50.1m +step 00301/01500 (20.07%) | loss: 2.570770 | lrm: 1.00 | dt: 2504.38ms | tok/sec: 209,348 | bf16_mfu: 31.96 | epoch: 1 pq: 3 rg: 56 | total time: 12.14m | eta: 50.0m +step 00302/01500 (20.13%) | loss: 2.582061 | lrm: 1.00 | dt: 2504.64ms | tok/sec: 209,326 | bf16_mfu: 31.96 | epoch: 1 pq: 3 rg: 56 | total time: 12.18m | eta: 50.0m +step 00303/01500 (20.20%) | loss: 2.599564 | lrm: 1.00 | dt: 2496.45ms | tok/sec: 210,013 | bf16_mfu: 32.06 | epoch: 1 pq: 3 rg: 56 | total time: 12.22m | eta: 49.9m +step 00304/01500 (20.27%) | loss: 2.617667 | lrm: 1.00 | dt: 2502.67ms | tok/sec: 209,491 | bf16_mfu: 31.98 | epoch: 1 pq: 3 rg: 56 | total time: 12.27m | eta: 49.9m +step 00305/01500 (20.33%) | loss: 2.607164 | lrm: 1.00 | dt: 2499.88ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 64 | total time: 12.31m | eta: 49.9m +step 00306/01500 (20.40%) | loss: 2.613627 | lrm: 1.00 | dt: 2509.25ms | tok/sec: 208,942 | bf16_mfu: 31.90 | epoch: 1 pq: 3 rg: 64 | total time: 12.35m | eta: 49.8m +step 00307/01500 (20.47%) | loss: 2.584020 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 3 rg: 64 | total time: 12.39m | eta: 49.8m +step 00308/01500 (20.53%) | loss: 2.639379 | lrm: 1.00 | dt: 2504.32ms | tok/sec: 209,353 | bf16_mfu: 31.96 | epoch: 1 pq: 3 rg: 64 | total time: 12.43m | eta: 49.7m +step 00309/01500 (20.60%) | loss: 2.666293 | lrm: 1.00 | dt: 2499.62ms | tok/sec: 209,746 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 64 | total time: 12.47m | eta: 49.7m +step 00310/01500 (20.67%) | loss: 2.689088 | lrm: 1.00 | dt: 2504.00ms | tok/sec: 209,379 | bf16_mfu: 31.97 | epoch: 1 pq: 3 rg: 64 | total time: 12.52m | eta: 49.6m +step 00311/01500 (20.73%) | loss: 2.672490 | lrm: 1.00 | dt: 2504.04ms | tok/sec: 209,377 | bf16_mfu: 31.97 | epoch: 1 pq: 3 rg: 64 | total time: 12.56m | eta: 49.6m +step 00312/01500 (20.80%) | loss: 2.624428 | lrm: 1.00 | dt: 2499.85ms | tok/sec: 209,727 | bf16_mfu: 32.02 | epoch: 1 pq: 3 rg: 64 | total time: 12.60m | eta: 49.6m +step 00313/01500 (20.87%) | loss: 2.686193 | lrm: 1.00 | dt: 2501.23ms | tok/sec: 209,612 | bf16_mfu: 32.00 | epoch: 1 pq: 3 rg: 64 | total time: 12.64m | eta: 49.5m +step 00314/01500 (20.93%) | loss: 2.658987 | lrm: 1.00 | dt: 2503.55ms | tok/sec: 209,418 | bf16_mfu: 31.97 | epoch: 1 pq: 3 rg: 72 | total time: 12.68m | eta: 49.5m +step 00315/01500 (21.00%) | loss: 2.668251 | lrm: 1.00 | dt: 2496.38ms | tok/sec: 210,019 | bf16_mfu: 32.06 | epoch: 1 pq: 3 rg: 72 | total time: 12.73m | eta: 49.4m +step 00316/01500 (21.07%) | loss: 2.661666 | lrm: 1.00 | dt: 2503.78ms | tok/sec: 209,398 | bf16_mfu: 31.97 | epoch: 1 pq: 3 rg: 72 | total time: 12.77m | eta: 49.4m +step 00317/01500 (21.13%) | loss: 2.642533 | lrm: 1.00 | dt: 2502.36ms | tok/sec: 209,517 | bf16_mfu: 31.99 | epoch: 1 pq: 3 rg: 72 | total time: 12.81m | eta: 49.4m +step 00318/01500 (21.20%) | loss: 2.658486 | lrm: 1.00 | dt: 2501.67ms | tok/sec: 209,575 | bf16_mfu: 32.00 | epoch: 1 pq: 3 rg: 72 | total time: 12.85m | eta: 49.3m +step 00319/01500 (21.27%) | loss: 2.665368 | lrm: 1.00 | dt: 2501.86ms | tok/sec: 209,558 | bf16_mfu: 31.99 | epoch: 1 pq: 3 rg: 72 | total time: 12.89m | eta: 49.3m +step 00320/01500 (21.33%) | loss: 2.652778 | lrm: 1.00 | dt: 2503.00ms | tok/sec: 209,463 | bf16_mfu: 31.98 | epoch: 1 pq: 3 rg: 72 | total time: 12.93m | eta: 49.2m +step 00321/01500 (21.40%) | loss: 2.658043 | lrm: 1.00 | dt: 2498.38ms | tok/sec: 209,851 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 72 | total time: 12.98m | eta: 49.2m +step 00322/01500 (21.47%) | loss: 2.681621 | lrm: 1.00 | dt: 2501.26ms | tok/sec: 209,609 | bf16_mfu: 32.00 | epoch: 1 pq: 3 rg: 80 | total time: 13.02m | eta: 49.1m +step 00323/01500 (21.53%) | loss: 2.668596 | lrm: 1.00 | dt: 2502.51ms | tok/sec: 209,504 | bf16_mfu: 31.99 | epoch: 1 pq: 3 rg: 80 | total time: 13.06m | eta: 49.1m +step 00324/01500 (21.60%) | loss: 2.632350 | lrm: 1.00 | dt: 2501.00ms | tok/sec: 209,631 | bf16_mfu: 32.01 | epoch: 1 pq: 3 rg: 80 | total time: 13.10m | eta: 49.1m +step 00325/01500 (21.67%) | loss: 2.624731 | lrm: 1.00 | dt: 2504.15ms | tok/sec: 209,367 | bf16_mfu: 31.97 | epoch: 1 pq: 3 rg: 80 | total time: 13.14m | eta: 49.0m +step 00326/01500 (21.73%) | loss: 2.591722 | lrm: 1.00 | dt: 2503.18ms | tok/sec: 209,448 | bf16_mfu: 31.98 | epoch: 1 pq: 3 rg: 80 | total time: 13.18m | eta: 49.0m +step 00327/01500 (21.80%) | loss: 2.547419 | lrm: 1.00 | dt: 2505.53ms | tok/sec: 209,252 | bf16_mfu: 31.95 | epoch: 1 pq: 3 rg: 80 | total time: 13.23m | eta: 48.9m +step 00328/01500 (21.87%) | loss: 2.578428 | lrm: 1.00 | dt: 2498.67ms | tok/sec: 209,827 | bf16_mfu: 32.04 | epoch: 1 pq: 3 rg: 80 | total time: 13.27m | eta: 48.9m +step 00329/01500 (21.93%) | loss: 2.560731 | lrm: 1.00 | dt: 2500.75ms | tok/sec: 209,652 | bf16_mfu: 32.01 | epoch: 1 pq: 4 rg: 0 | total time: 13.31m | eta: 48.9m +step 00330/01500 (22.00%) | loss: 2.585317 | lrm: 1.00 | dt: 2497.66ms | tok/sec: 209,912 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 0 | total time: 13.35m | eta: 48.8m +step 00331/01500 (22.07%) | loss: 2.612563 | lrm: 1.00 | dt: 2501.03ms | tok/sec: 209,628 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 0 | total time: 13.39m | eta: 48.8m +step 00332/01500 (22.13%) | loss: 2.622800 | lrm: 1.00 | dt: 2499.40ms | tok/sec: 209,765 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 0 | total time: 13.43m | eta: 48.7m +step 00333/01500 (22.20%) | loss: 2.609684 | lrm: 1.00 | dt: 2503.44ms | tok/sec: 209,426 | bf16_mfu: 31.97 | epoch: 1 pq: 4 rg: 0 | total time: 13.48m | eta: 48.7m +step 00334/01500 (22.27%) | loss: 2.639728 | lrm: 1.00 | dt: 2501.09ms | tok/sec: 209,624 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 0 | total time: 13.52m | eta: 48.6m +step 00335/01500 (22.33%) | loss: 2.655841 | lrm: 1.00 | dt: 2496.66ms | tok/sec: 209,995 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 0 | total time: 13.56m | eta: 48.6m +step 00336/01500 (22.40%) | loss: 2.605947 | lrm: 1.00 | dt: 2501.92ms | tok/sec: 209,554 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 0 | total time: 13.60m | eta: 48.6m +step 00337/01500 (22.47%) | loss: 2.620740 | lrm: 1.00 | dt: 2503.39ms | tok/sec: 209,431 | bf16_mfu: 31.97 | epoch: 1 pq: 4 rg: 0 | total time: 13.64m | eta: 48.5m +step 00338/01500 (22.53%) | loss: 2.611902 | lrm: 1.00 | dt: 2496.71ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 8 | total time: 13.68m | eta: 48.5m +step 00339/01500 (22.60%) | loss: 2.629633 | lrm: 1.00 | dt: 2504.56ms | tok/sec: 209,333 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 8 | total time: 13.73m | eta: 48.4m +step 00340/01500 (22.67%) | loss: 2.595688 | lrm: 1.00 | dt: 2498.92ms | tok/sec: 209,805 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 8 | total time: 13.77m | eta: 48.4m +step 00341/01500 (22.73%) | loss: 2.599709 | lrm: 1.00 | dt: 2501.73ms | tok/sec: 209,570 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 8 | total time: 13.81m | eta: 48.4m +step 00342/01500 (22.80%) | loss: 2.642176 | lrm: 1.00 | dt: 2498.07ms | tok/sec: 209,877 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 8 | total time: 13.85m | eta: 48.3m +step 00343/01500 (22.87%) | loss: 2.618690 | lrm: 1.00 | dt: 2497.95ms | tok/sec: 209,886 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 8 | total time: 13.89m | eta: 48.3m +step 00344/01500 (22.93%) | loss: 2.640408 | lrm: 1.00 | dt: 2500.23ms | tok/sec: 209,696 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 8 | total time: 13.93m | eta: 48.2m +step 00345/01500 (23.00%) | loss: 2.651068 | lrm: 1.00 | dt: 2499.25ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 8 | total time: 13.98m | eta: 48.2m +step 00346/01500 (23.07%) | loss: 2.640402 | lrm: 1.00 | dt: 2496.41ms | tok/sec: 210,016 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 16 | total time: 14.02m | eta: 48.1m +step 00347/01500 (23.13%) | loss: 2.640979 | lrm: 1.00 | dt: 2496.47ms | tok/sec: 210,011 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 16 | total time: 14.06m | eta: 48.1m +step 00348/01500 (23.20%) | loss: 2.625766 | lrm: 1.00 | dt: 2505.88ms | tok/sec: 209,223 | bf16_mfu: 31.94 | epoch: 1 pq: 4 rg: 16 | total time: 14.10m | eta: 48.1m +step 00349/01500 (23.27%) | loss: 2.638305 | lrm: 1.00 | dt: 2504.49ms | tok/sec: 209,339 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 16 | total time: 14.14m | eta: 48.0m +step 00350/01500 (23.33%) | loss: 2.631730 | lrm: 1.00 | dt: 2502.65ms | tok/sec: 209,492 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 16 | total time: 14.18m | eta: 48.0m +step 00351/01500 (23.40%) | loss: 2.678804 | lrm: 1.00 | dt: 2499.11ms | tok/sec: 209,789 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 16 | total time: 14.23m | eta: 47.9m +step 00352/01500 (23.47%) | loss: 2.662850 | lrm: 1.00 | dt: 2499.22ms | tok/sec: 209,780 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 16 | total time: 14.27m | eta: 47.9m +step 00353/01500 (23.53%) | loss: 2.642324 | lrm: 1.00 | dt: 2500.14ms | tok/sec: 209,703 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 16 | total time: 14.31m | eta: 47.8m +step 00354/01500 (23.60%) | loss: 2.685310 | lrm: 1.00 | dt: 2504.34ms | tok/sec: 209,351 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 24 | total time: 14.35m | eta: 47.8m +step 00355/01500 (23.67%) | loss: 2.695756 | lrm: 1.00 | dt: 2500.28ms | tok/sec: 209,692 | bf16_mfu: 32.01 | epoch: 1 pq: 4 rg: 24 | total time: 14.39m | eta: 47.8m +step 00356/01500 (23.73%) | loss: 2.614161 | lrm: 1.00 | dt: 2501.99ms | tok/sec: 209,548 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 24 | total time: 14.43m | eta: 47.7m +step 00357/01500 (23.80%) | loss: 2.627071 | lrm: 1.00 | dt: 2500.09ms | tok/sec: 209,707 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 24 | total time: 14.48m | eta: 47.7m +step 00358/01500 (23.87%) | loss: 2.638768 | lrm: 1.00 | dt: 2497.49ms | tok/sec: 209,926 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 24 | total time: 14.52m | eta: 47.6m +step 00359/01500 (23.93%) | loss: 2.608650 | lrm: 1.00 | dt: 2499.26ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 24 | total time: 14.56m | eta: 47.6m +step 00360/01500 (24.00%) | loss: 2.622549 | lrm: 1.00 | dt: 2499.18ms | tok/sec: 209,783 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 24 | total time: 14.60m | eta: 47.6m +step 00361/01500 (24.07%) | loss: 2.607933 | lrm: 1.00 | dt: 2496.72ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 24 | total time: 14.64m | eta: 47.5m +step 00362/01500 (24.13%) | loss: 2.607696 | lrm: 1.00 | dt: 2496.08ms | tok/sec: 210,044 | bf16_mfu: 32.07 | epoch: 1 pq: 4 rg: 24 | total time: 14.68m | eta: 47.5m +step 00363/01500 (24.20%) | loss: 2.595557 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 32 | total time: 14.73m | eta: 47.4m +step 00364/01500 (24.27%) | loss: 2.650946 | lrm: 1.00 | dt: 2498.88ms | tok/sec: 209,809 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 32 | total time: 14.77m | eta: 47.4m +step 00365/01500 (24.33%) | loss: 2.641341 | lrm: 1.00 | dt: 2502.68ms | tok/sec: 209,490 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 32 | total time: 14.81m | eta: 47.3m +step 00366/01500 (24.40%) | loss: 2.646917 | lrm: 1.00 | dt: 2502.62ms | tok/sec: 209,495 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 32 | total time: 14.85m | eta: 47.3m +step 00367/01500 (24.47%) | loss: 2.635313 | lrm: 1.00 | dt: 2496.63ms | tok/sec: 209,998 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 32 | total time: 14.89m | eta: 47.3m +step 00368/01500 (24.53%) | loss: 2.604769 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,811 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 32 | total time: 14.93m | eta: 47.2m +step 00369/01500 (24.60%) | loss: 2.614996 | lrm: 1.00 | dt: 2497.13ms | tok/sec: 209,956 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 32 | total time: 14.98m | eta: 47.2m +step 00370/01500 (24.67%) | loss: 2.614470 | lrm: 1.00 | dt: 2499.68ms | tok/sec: 209,742 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 32 | total time: 15.02m | eta: 47.1m +step 00371/01500 (24.73%) | loss: 2.626087 | lrm: 1.00 | dt: 2503.00ms | tok/sec: 209,463 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 40 | total time: 15.06m | eta: 47.1m +step 00372/01500 (24.80%) | loss: 2.616861 | lrm: 1.00 | dt: 2502.16ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 40 | total time: 15.10m | eta: 47.1m +step 00373/01500 (24.87%) | loss: 2.583940 | lrm: 1.00 | dt: 2499.57ms | tok/sec: 209,751 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 40 | total time: 15.14m | eta: 47.0m +step 00374/01500 (24.93%) | loss: 2.535830 | lrm: 1.00 | dt: 2499.88ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 40 | total time: 15.18m | eta: 47.0m +step 00375/01500 (25.00%) | loss: 2.537096 | lrm: 1.00 | dt: 2502.59ms | tok/sec: 209,498 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 40 | total time: 15.23m | eta: 46.9m +step 00376/01500 (25.07%) | loss: 2.543708 | lrm: 1.00 | dt: 2497.40ms | tok/sec: 209,933 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 40 | total time: 15.27m | eta: 46.9m +step 00377/01500 (25.13%) | loss: 2.540690 | lrm: 1.00 | dt: 2499.48ms | tok/sec: 209,758 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 40 | total time: 15.31m | eta: 46.8m +step 00378/01500 (25.20%) | loss: 2.534647 | lrm: 1.00 | dt: 2500.09ms | tok/sec: 209,707 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 40 | total time: 15.35m | eta: 46.8m +step 00379/01500 (25.27%) | loss: 2.458322 | lrm: 1.00 | dt: 3008.53ms | tok/sec: 174,266 | bf16_mfu: 26.61 | epoch: 1 pq: 4 rg: 48 | total time: 15.40m | eta: 46.8m +step 00380/01500 (25.33%) | loss: 2.478253 | lrm: 1.00 | dt: 2498.42ms | tok/sec: 209,847 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 48 | total time: 15.44m | eta: 46.7m +step 00381/01500 (25.40%) | loss: 2.439136 | lrm: 1.00 | dt: 2497.76ms | tok/sec: 209,903 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 48 | total time: 15.48m | eta: 46.7m +step 00382/01500 (25.47%) | loss: 2.367838 | lrm: 1.00 | dt: 2499.43ms | tok/sec: 209,762 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 48 | total time: 15.53m | eta: 46.7m +step 00383/01500 (25.53%) | loss: 2.388532 | lrm: 1.00 | dt: 2498.49ms | tok/sec: 209,842 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 48 | total time: 15.57m | eta: 46.6m +step 00384/01500 (25.60%) | loss: 2.437780 | lrm: 1.00 | dt: 2500.15ms | tok/sec: 209,702 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 48 | total time: 15.61m | eta: 46.6m +step 00385/01500 (25.67%) | loss: 2.414677 | lrm: 1.00 | dt: 2497.87ms | tok/sec: 209,893 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 48 | total time: 15.65m | eta: 46.5m +step 00386/01500 (25.73%) | loss: 2.424039 | lrm: 1.00 | dt: 2502.17ms | tok/sec: 209,533 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 48 | total time: 15.69m | eta: 46.5m +step 00387/01500 (25.80%) | loss: 2.392342 | lrm: 1.00 | dt: 2495.37ms | tok/sec: 210,104 | bf16_mfu: 32.08 | epoch: 1 pq: 4 rg: 56 | total time: 15.73m | eta: 46.5m +step 00388/01500 (25.87%) | loss: 2.331812 | lrm: 1.00 | dt: 2497.23ms | tok/sec: 209,947 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 56 | total time: 15.78m | eta: 46.4m +step 00389/01500 (25.93%) | loss: 2.366487 | lrm: 1.00 | dt: 2502.65ms | tok/sec: 209,493 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 56 | total time: 15.82m | eta: 46.4m +step 00390/01500 (26.00%) | loss: 2.327390 | lrm: 1.00 | dt: 2497.37ms | tok/sec: 209,935 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 56 | total time: 15.86m | eta: 46.3m +step 00391/01500 (26.07%) | loss: 2.420736 | lrm: 1.00 | dt: 2498.26ms | tok/sec: 209,861 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 56 | total time: 15.90m | eta: 46.3m +step 00392/01500 (26.13%) | loss: 2.435264 | lrm: 1.00 | dt: 2501.12ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 56 | total time: 15.94m | eta: 46.2m +step 00393/01500 (26.20%) | loss: 2.430283 | lrm: 1.00 | dt: 2502.73ms | tok/sec: 209,486 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 56 | total time: 15.98m | eta: 46.2m +step 00394/01500 (26.27%) | loss: 2.462935 | lrm: 1.00 | dt: 2497.48ms | tok/sec: 209,927 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 56 | total time: 16.03m | eta: 46.2m +step 00395/01500 (26.33%) | loss: 2.477935 | lrm: 1.00 | dt: 2504.88ms | tok/sec: 209,306 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 56 | total time: 16.07m | eta: 46.1m +step 00396/01500 (26.40%) | loss: 2.477937 | lrm: 1.00 | dt: 2499.35ms | tok/sec: 209,769 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 64 | total time: 16.11m | eta: 46.1m +step 00397/01500 (26.47%) | loss: 2.507707 | lrm: 1.00 | dt: 2505.31ms | tok/sec: 209,270 | bf16_mfu: 31.95 | epoch: 1 pq: 4 rg: 64 | total time: 16.15m | eta: 46.0m +step 00398/01500 (26.53%) | loss: 2.508839 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 64 | total time: 16.19m | eta: 46.0m +step 00399/01500 (26.60%) | loss: 2.461043 | lrm: 1.00 | dt: 2498.06ms | tok/sec: 209,877 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 64 | total time: 16.23m | eta: 45.9m +Step 00400 | Validation bpb: 0.769512 +Step 00400 | ICL early: 2.8437 late: 2.9914 score: +0.1477 | Induction loss: 10.8087 acc: 0.0461 (random-half: 14.1634) +step 00400/01500 (26.67%) | loss: 2.516207 | lrm: 1.00 | dt: 2497.98ms | tok/sec: 209,885 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 64 | total time: 16.28m | eta: 45.9m +step 00401/01500 (26.73%) | loss: 2.519900 | lrm: 1.00 | dt: 2504.08ms | tok/sec: 209,373 | bf16_mfu: 31.97 | epoch: 1 pq: 4 rg: 64 | total time: 16.32m | eta: 45.9m +step 00402/01500 (26.80%) | loss: 2.501173 | lrm: 1.00 | dt: 2502.29ms | tok/sec: 209,523 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 64 | total time: 16.36m | eta: 45.8m +step 00403/01500 (26.87%) | loss: 2.533427 | lrm: 1.00 | dt: 2501.77ms | tok/sec: 209,566 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 64 | total time: 16.40m | eta: 45.8m +step 00404/01500 (26.93%) | loss: 2.571604 | lrm: 1.00 | dt: 2503.99ms | tok/sec: 209,381 | bf16_mfu: 31.97 | epoch: 1 pq: 4 rg: 72 | total time: 16.44m | eta: 45.7m +step 00405/01500 (27.00%) | loss: 2.573077 | lrm: 1.00 | dt: 2500.14ms | tok/sec: 209,703 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 72 | total time: 16.48m | eta: 45.7m +step 00406/01500 (27.07%) | loss: 2.582443 | lrm: 1.00 | dt: 2504.71ms | tok/sec: 209,321 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 72 | total time: 16.53m | eta: 45.7m +step 00407/01500 (27.13%) | loss: 2.595983 | lrm: 1.00 | dt: 2499.64ms | tok/sec: 209,745 | bf16_mfu: 32.02 | epoch: 1 pq: 4 rg: 72 | total time: 16.57m | eta: 45.6m +step 00408/01500 (27.20%) | loss: 2.582707 | lrm: 1.00 | dt: 2502.59ms | tok/sec: 209,498 | bf16_mfu: 31.99 | epoch: 1 pq: 4 rg: 72 | total time: 16.61m | eta: 45.6m +step 00409/01500 (27.27%) | loss: 2.591550 | lrm: 1.00 | dt: 2497.45ms | tok/sec: 209,929 | bf16_mfu: 32.05 | epoch: 1 pq: 4 rg: 72 | total time: 16.65m | eta: 45.5m +step 00410/01500 (27.33%) | loss: 2.597649 | lrm: 1.00 | dt: 2500.90ms | tok/sec: 209,640 | bf16_mfu: 32.01 | epoch: 1 pq: 4 rg: 72 | total time: 16.69m | eta: 45.5m +step 00411/01500 (27.40%) | loss: 2.615967 | lrm: 1.00 | dt: 2501.65ms | tok/sec: 209,576 | bf16_mfu: 32.00 | epoch: 1 pq: 4 rg: 72 | total time: 16.73m | eta: 45.4m +step 00412/01500 (27.47%) | loss: 2.592289 | lrm: 1.00 | dt: 2498.68ms | tok/sec: 209,826 | bf16_mfu: 32.04 | epoch: 1 pq: 4 rg: 80 | total time: 16.78m | eta: 45.4m +step 00413/01500 (27.53%) | loss: 2.633285 | lrm: 1.00 | dt: 2503.42ms | tok/sec: 209,428 | bf16_mfu: 31.97 | epoch: 1 pq: 4 rg: 80 | total time: 16.82m | eta: 45.4m +step 00414/01500 (27.60%) | loss: 2.634857 | lrm: 1.00 | dt: 2502.98ms | tok/sec: 209,465 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 80 | total time: 16.86m | eta: 45.3m +step 00415/01500 (27.67%) | loss: 2.646015 | lrm: 1.00 | dt: 2496.70ms | tok/sec: 209,992 | bf16_mfu: 32.06 | epoch: 1 pq: 4 rg: 80 | total time: 16.90m | eta: 45.3m +step 00416/01500 (27.73%) | loss: 2.659769 | lrm: 1.00 | dt: 2498.94ms | tok/sec: 209,804 | bf16_mfu: 32.03 | epoch: 1 pq: 4 rg: 80 | total time: 16.94m | eta: 45.2m +step 00417/01500 (27.80%) | loss: 2.644014 | lrm: 1.00 | dt: 2504.60ms | tok/sec: 209,329 | bf16_mfu: 31.96 | epoch: 1 pq: 4 rg: 80 | total time: 16.98m | eta: 45.2m +step 00418/01500 (27.87%) | loss: 2.642324 | lrm: 1.00 | dt: 2503.14ms | tok/sec: 209,452 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 80 | total time: 17.03m | eta: 45.2m +step 00419/01500 (27.93%) | loss: 2.610305 | lrm: 1.00 | dt: 2503.30ms | tok/sec: 209,438 | bf16_mfu: 31.98 | epoch: 1 pq: 4 rg: 80 | total time: 17.07m | eta: 45.1m +step 00420/01500 (28.00%) | loss: 2.601027 | lrm: 1.00 | dt: 2500.63ms | tok/sec: 209,662 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 0 | total time: 17.11m | eta: 45.1m +step 00421/01500 (28.07%) | loss: 2.611735 | lrm: 1.00 | dt: 2502.53ms | tok/sec: 209,503 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 0 | total time: 17.15m | eta: 45.0m +step 00422/01500 (28.13%) | loss: 2.596965 | lrm: 1.00 | dt: 2500.54ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 0 | total time: 17.19m | eta: 45.0m +step 00423/01500 (28.20%) | loss: 2.610565 | lrm: 1.00 | dt: 2498.29ms | tok/sec: 209,858 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 0 | total time: 17.23m | eta: 44.9m +step 00424/01500 (28.27%) | loss: 2.614224 | lrm: 1.00 | dt: 2498.66ms | tok/sec: 209,828 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 0 | total time: 17.28m | eta: 44.9m +step 00425/01500 (28.33%) | loss: 2.633454 | lrm: 1.00 | dt: 2500.31ms | tok/sec: 209,689 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 0 | total time: 17.32m | eta: 44.9m +step 00426/01500 (28.40%) | loss: 2.653873 | lrm: 1.00 | dt: 2495.96ms | tok/sec: 210,054 | bf16_mfu: 32.07 | epoch: 1 pq: 5 rg: 0 | total time: 17.36m | eta: 44.8m +step 00427/01500 (28.47%) | loss: 2.624991 | lrm: 1.00 | dt: 2498.74ms | tok/sec: 209,821 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 0 | total time: 17.40m | eta: 44.8m +step 00428/01500 (28.53%) | loss: 2.600636 | lrm: 1.00 | dt: 2504.57ms | tok/sec: 209,332 | bf16_mfu: 31.96 | epoch: 1 pq: 5 rg: 8 | total time: 17.44m | eta: 44.7m +step 00429/01500 (28.60%) | loss: 2.589331 | lrm: 1.00 | dt: 2496.92ms | tok/sec: 209,973 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 8 | total time: 17.48m | eta: 44.7m +step 00430/01500 (28.67%) | loss: 2.603567 | lrm: 1.00 | dt: 2502.68ms | tok/sec: 209,490 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 8 | total time: 17.53m | eta: 44.6m +step 00431/01500 (28.73%) | loss: 2.654011 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 8 | total time: 17.57m | eta: 44.6m +step 00432/01500 (28.80%) | loss: 2.552250 | lrm: 1.00 | dt: 2499.25ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 8 | total time: 17.61m | eta: 44.6m +step 00433/01500 (28.87%) | loss: 2.576312 | lrm: 1.00 | dt: 2501.86ms | tok/sec: 209,559 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 8 | total time: 17.65m | eta: 44.5m +step 00434/01500 (28.93%) | loss: 2.587704 | lrm: 1.00 | dt: 2500.07ms | tok/sec: 209,709 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 8 | total time: 17.69m | eta: 44.5m +step 00435/01500 (29.00%) | loss: 2.696486 | lrm: 1.00 | dt: 2502.16ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 8 | total time: 17.73m | eta: 44.4m +step 00436/01500 (29.07%) | loss: 2.713219 | lrm: 1.00 | dt: 2498.66ms | tok/sec: 209,827 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 8 | total time: 17.78m | eta: 44.4m +step 00437/01500 (29.13%) | loss: 2.742426 | lrm: 1.00 | dt: 2500.29ms | tok/sec: 209,691 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 16 | total time: 17.82m | eta: 44.4m +step 00438/01500 (29.20%) | loss: 2.716522 | lrm: 1.00 | dt: 2502.57ms | tok/sec: 209,499 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 16 | total time: 17.86m | eta: 44.3m +step 00439/01500 (29.27%) | loss: 2.714635 | lrm: 1.00 | dt: 2499.53ms | tok/sec: 209,754 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 16 | total time: 17.90m | eta: 44.3m +step 00440/01500 (29.33%) | loss: 2.735734 | lrm: 1.00 | dt: 2503.85ms | tok/sec: 209,392 | bf16_mfu: 31.97 | epoch: 1 pq: 5 rg: 16 | total time: 17.94m | eta: 44.2m +step 00441/01500 (29.40%) | loss: 2.716592 | lrm: 1.00 | dt: 2502.81ms | tok/sec: 209,479 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 16 | total time: 17.98m | eta: 44.2m +step 00442/01500 (29.47%) | loss: 2.679167 | lrm: 1.00 | dt: 2504.39ms | tok/sec: 209,347 | bf16_mfu: 31.96 | epoch: 1 pq: 5 rg: 16 | total time: 18.03m | eta: 44.1m +step 00443/01500 (29.53%) | loss: 2.652363 | lrm: 1.00 | dt: 2503.08ms | tok/sec: 209,456 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 16 | total time: 18.07m | eta: 44.1m +step 00444/01500 (29.60%) | loss: 2.668071 | lrm: 1.00 | dt: 2496.79ms | tok/sec: 209,984 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 16 | total time: 18.11m | eta: 44.1m +step 00445/01500 (29.67%) | loss: 2.665459 | lrm: 1.00 | dt: 2502.05ms | tok/sec: 209,543 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 24 | total time: 18.15m | eta: 44.0m +step 00446/01500 (29.73%) | loss: 2.679841 | lrm: 1.00 | dt: 2501.16ms | tok/sec: 209,617 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 24 | total time: 18.19m | eta: 44.0m +step 00447/01500 (29.80%) | loss: 2.669771 | lrm: 1.00 | dt: 2502.88ms | tok/sec: 209,473 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 24 | total time: 18.23m | eta: 43.9m +step 00448/01500 (29.87%) | loss: 2.672466 | lrm: 1.00 | dt: 2500.79ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 24 | total time: 18.28m | eta: 43.9m +step 00449/01500 (29.93%) | loss: 2.641235 | lrm: 1.00 | dt: 2503.18ms | tok/sec: 209,448 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 24 | total time: 18.32m | eta: 43.9m +step 00450/01500 (30.00%) | loss: 2.633718 | lrm: 1.00 | dt: 2502.95ms | tok/sec: 209,467 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 24 | total time: 18.36m | eta: 43.8m +step 00451/01500 (30.07%) | loss: 2.610007 | lrm: 1.00 | dt: 2504.55ms | tok/sec: 209,334 | bf16_mfu: 31.96 | epoch: 1 pq: 5 rg: 24 | total time: 18.40m | eta: 43.8m +step 00452/01500 (30.13%) | loss: 2.558630 | lrm: 1.00 | dt: 2494.24ms | tok/sec: 210,199 | bf16_mfu: 32.09 | epoch: 1 pq: 5 rg: 24 | total time: 18.44m | eta: 43.7m +step 00453/01500 (30.20%) | loss: 2.532510 | lrm: 1.00 | dt: 2500.48ms | tok/sec: 209,674 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 32 | total time: 18.48m | eta: 43.7m +step 00454/01500 (30.27%) | loss: 2.527563 | lrm: 1.00 | dt: 2501.22ms | tok/sec: 209,612 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 32 | total time: 18.53m | eta: 43.6m +step 00455/01500 (30.33%) | loss: 2.534919 | lrm: 1.00 | dt: 2504.73ms | tok/sec: 209,319 | bf16_mfu: 31.96 | epoch: 1 pq: 5 rg: 32 | total time: 18.57m | eta: 43.6m +step 00456/01500 (30.40%) | loss: 2.523789 | lrm: 1.00 | dt: 2499.57ms | tok/sec: 209,750 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 32 | total time: 18.61m | eta: 43.6m +step 00457/01500 (30.47%) | loss: 2.497783 | lrm: 1.00 | dt: 2501.66ms | tok/sec: 209,575 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 32 | total time: 18.65m | eta: 43.5m +step 00458/01500 (30.53%) | loss: 2.515548 | lrm: 1.00 | dt: 2502.16ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 32 | total time: 18.69m | eta: 43.5m +step 00459/01500 (30.60%) | loss: 2.558479 | lrm: 1.00 | dt: 2500.55ms | tok/sec: 209,668 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 32 | total time: 18.74m | eta: 43.4m +step 00460/01500 (30.67%) | loss: 2.568665 | lrm: 1.00 | dt: 2601.98ms | tok/sec: 201,495 | bf16_mfu: 30.76 | epoch: 1 pq: 5 rg: 32 | total time: 18.78m | eta: 43.4m +step 00461/01500 (30.73%) | loss: 2.549469 | lrm: 1.00 | dt: 2497.65ms | tok/sec: 209,912 | bf16_mfu: 32.05 | epoch: 1 pq: 5 rg: 40 | total time: 18.82m | eta: 43.4m +step 00462/01500 (30.80%) | loss: 2.527152 | lrm: 1.00 | dt: 2497.69ms | tok/sec: 209,908 | bf16_mfu: 32.05 | epoch: 1 pq: 5 rg: 40 | total time: 18.86m | eta: 43.3m +step 00463/01500 (30.87%) | loss: 2.562973 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 40 | total time: 18.90m | eta: 43.3m +step 00464/01500 (30.93%) | loss: 2.558161 | lrm: 1.00 | dt: 2496.52ms | tok/sec: 210,007 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 40 | total time: 18.94m | eta: 43.2m +step 00465/01500 (31.00%) | loss: 2.654085 | lrm: 1.00 | dt: 2503.38ms | tok/sec: 209,432 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 40 | total time: 18.99m | eta: 43.2m +step 00466/01500 (31.07%) | loss: 2.631398 | lrm: 1.00 | dt: 2498.73ms | tok/sec: 209,821 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 40 | total time: 19.03m | eta: 43.1m +step 00467/01500 (31.13%) | loss: 2.654613 | lrm: 1.00 | dt: 2506.62ms | tok/sec: 209,161 | bf16_mfu: 31.93 | epoch: 1 pq: 5 rg: 40 | total time: 19.07m | eta: 43.1m +step 00468/01500 (31.20%) | loss: 2.650343 | lrm: 1.00 | dt: 2500.46ms | tok/sec: 209,676 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 40 | total time: 19.11m | eta: 43.1m +step 00469/01500 (31.27%) | loss: 2.615141 | lrm: 1.00 | dt: 2498.96ms | tok/sec: 209,802 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 48 | total time: 19.15m | eta: 43.0m +step 00470/01500 (31.33%) | loss: 2.604841 | lrm: 1.00 | dt: 2500.30ms | tok/sec: 209,689 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 48 | total time: 19.20m | eta: 43.0m +step 00471/01500 (31.40%) | loss: 2.579015 | lrm: 1.00 | dt: 2496.60ms | tok/sec: 210,001 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 48 | total time: 19.24m | eta: 42.9m +step 00472/01500 (31.47%) | loss: 2.538458 | lrm: 1.00 | dt: 2496.98ms | tok/sec: 209,968 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 48 | total time: 19.28m | eta: 42.9m +step 00473/01500 (31.53%) | loss: 2.529463 | lrm: 1.00 | dt: 2498.70ms | tok/sec: 209,824 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 48 | total time: 19.32m | eta: 42.9m +step 00474/01500 (31.60%) | loss: 2.578395 | lrm: 1.00 | dt: 2499.59ms | tok/sec: 209,749 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 48 | total time: 19.36m | eta: 42.8m +step 00475/01500 (31.67%) | loss: 2.591929 | lrm: 1.00 | dt: 2501.67ms | tok/sec: 209,575 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 48 | total time: 19.40m | eta: 42.8m +step 00476/01500 (31.73%) | loss: 2.588667 | lrm: 1.00 | dt: 2504.15ms | tok/sec: 209,367 | bf16_mfu: 31.97 | epoch: 1 pq: 5 rg: 48 | total time: 19.45m | eta: 42.7m +step 00477/01500 (31.80%) | loss: 2.549180 | lrm: 1.00 | dt: 2498.01ms | tok/sec: 209,882 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 48 | total time: 19.49m | eta: 42.7m +step 00478/01500 (31.87%) | loss: 2.571854 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 56 | total time: 19.53m | eta: 42.6m +step 00479/01500 (31.93%) | loss: 2.581863 | lrm: 1.00 | dt: 2495.07ms | tok/sec: 210,129 | bf16_mfu: 32.08 | epoch: 1 pq: 5 rg: 56 | total time: 19.57m | eta: 42.6m +step 00480/01500 (32.00%) | loss: 2.540622 | lrm: 1.00 | dt: 2502.05ms | tok/sec: 209,543 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 56 | total time: 19.61m | eta: 42.6m +step 00481/01500 (32.07%) | loss: 2.452574 | lrm: 1.00 | dt: 2504.77ms | tok/sec: 209,315 | bf16_mfu: 31.96 | epoch: 1 pq: 5 rg: 56 | total time: 19.65m | eta: 42.5m +step 00482/01500 (32.13%) | loss: 2.525343 | lrm: 1.00 | dt: 2500.68ms | tok/sec: 209,658 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 56 | total time: 19.70m | eta: 42.5m +step 00483/01500 (32.20%) | loss: 2.550240 | lrm: 1.00 | dt: 2496.04ms | tok/sec: 210,048 | bf16_mfu: 32.07 | epoch: 1 pq: 5 rg: 56 | total time: 19.74m | eta: 42.4m +step 00484/01500 (32.27%) | loss: 2.497811 | lrm: 1.00 | dt: 2498.28ms | tok/sec: 209,859 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 56 | total time: 19.78m | eta: 42.4m +step 00485/01500 (32.33%) | loss: 2.496545 | lrm: 1.00 | dt: 2499.34ms | tok/sec: 209,770 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 56 | total time: 19.82m | eta: 42.4m +step 00486/01500 (32.40%) | loss: 2.525746 | lrm: 1.00 | dt: 2503.49ms | tok/sec: 209,423 | bf16_mfu: 31.97 | epoch: 1 pq: 5 rg: 64 | total time: 19.86m | eta: 42.3m +step 00487/01500 (32.47%) | loss: 2.539668 | lrm: 1.00 | dt: 2497.55ms | tok/sec: 209,921 | bf16_mfu: 32.05 | epoch: 1 pq: 5 rg: 64 | total time: 19.90m | eta: 42.3m +step 00488/01500 (32.53%) | loss: 2.530516 | lrm: 1.00 | dt: 2500.24ms | tok/sec: 209,694 | bf16_mfu: 32.02 | epoch: 1 pq: 5 rg: 64 | total time: 19.94m | eta: 42.2m +step 00489/01500 (32.60%) | loss: 2.537329 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 5 rg: 64 | total time: 19.99m | eta: 42.2m +step 00490/01500 (32.67%) | loss: 2.534464 | lrm: 1.00 | dt: 2503.14ms | tok/sec: 209,452 | bf16_mfu: 31.98 | epoch: 1 pq: 5 rg: 64 | total time: 20.03m | eta: 42.1m +step 00491/01500 (32.73%) | loss: 2.530831 | lrm: 1.00 | dt: 2503.65ms | tok/sec: 209,409 | bf16_mfu: 31.97 | epoch: 1 pq: 5 rg: 64 | total time: 20.07m | eta: 42.1m +step 00492/01500 (32.80%) | loss: 2.498196 | lrm: 1.00 | dt: 2500.75ms | tok/sec: 209,652 | bf16_mfu: 32.01 | epoch: 1 pq: 5 rg: 64 | total time: 20.11m | eta: 42.1m +step 00493/01500 (32.87%) | loss: 2.483597 | lrm: 1.00 | dt: 2497.92ms | tok/sec: 209,889 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 64 | total time: 20.15m | eta: 42.0m +step 00494/01500 (32.93%) | loss: 2.509084 | lrm: 1.00 | dt: 2502.00ms | tok/sec: 209,547 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 72 | total time: 20.20m | eta: 42.0m +step 00495/01500 (33.00%) | loss: 2.470687 | lrm: 1.00 | dt: 2506.07ms | tok/sec: 209,207 | bf16_mfu: 31.94 | epoch: 1 pq: 5 rg: 72 | total time: 20.24m | eta: 41.9m +step 00496/01500 (33.07%) | loss: 2.479860 | lrm: 1.00 | dt: 2502.48ms | tok/sec: 209,507 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 72 | total time: 20.28m | eta: 41.9m +step 00497/01500 (33.13%) | loss: 2.529020 | lrm: 1.00 | dt: 2496.55ms | tok/sec: 210,005 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 72 | total time: 20.32m | eta: 41.9m +step 00498/01500 (33.20%) | loss: 2.506069 | lrm: 1.00 | dt: 2496.61ms | tok/sec: 209,999 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 72 | total time: 20.36m | eta: 41.8m +step 00499/01500 (33.27%) | loss: 2.503669 | lrm: 1.00 | dt: 2502.22ms | tok/sec: 209,528 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 72 | total time: 20.40m | eta: 41.8m +Step 00500 | Validation bpb: 0.758676 +Step 00500 | ICL early: 2.7840 late: 2.9436 score: +0.1596 | Induction loss: 10.4020 acc: 0.0576 (random-half: 14.0550) +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au. The chemical symbol of silver is Ag. The chemical symbol of copper is +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. Today is Saturday, and tomorrow is Sunday. If today is Saturday, +<|bos|>The opposite of hot is cold. Hot is a color that is associated with the sun and the sun's +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like to wear blue jeans and I like to wear blue jeans. +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +2026-08-10 21:53:12,881 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank5.pt +2026-08-10 21:53:14,092 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank3.pt +2026-08-10 21:53:17,130 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank4.pt +2026-08-10 21:53:18,463 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank1.pt +2026-08-10 21:53:18,846 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank7.pt +2026-08-10 21:53:19,073 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank2.pt +2026-08-10 21:53:19,269 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank6.pt +2026-08-10 21:54:18,230 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_000500.pt +2026-08-10 21:54:18,232 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_000500.json +2026-08-10 21:54:25,608 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000500_rank0.pt +step 00500/01500 (33.33%) | loss: 2.500904 | lrm: 1.00 | dt: 2475.99ms | tok/sec: 211,748 | bf16_mfu: 32.33 | epoch: 1 pq: 5 rg: 72 | total time: 20.44m | eta: 41.7m +step 00501/01500 (33.40%) | loss: 2.463526 | lrm: 1.00 | dt: 2481.54ms | tok/sec: 211,275 | bf16_mfu: 32.26 | epoch: 1 pq: 5 rg: 72 | total time: 20.49m | eta: 41.7m +step 00502/01500 (33.47%) | loss: 2.518413 | lrm: 1.00 | dt: 2484.65ms | tok/sec: 211,011 | bf16_mfu: 32.22 | epoch: 1 pq: 5 rg: 80 | total time: 20.53m | eta: 41.6m +step 00503/01500 (33.53%) | loss: 2.577312 | lrm: 1.00 | dt: 2492.33ms | tok/sec: 210,360 | bf16_mfu: 32.12 | epoch: 1 pq: 5 rg: 80 | total time: 20.57m | eta: 41.6m +step 00504/01500 (33.60%) | loss: 2.613699 | lrm: 1.00 | dt: 2491.09ms | tok/sec: 210,465 | bf16_mfu: 32.13 | epoch: 1 pq: 5 rg: 80 | total time: 20.61m | eta: 41.6m +step 00505/01500 (33.67%) | loss: 2.619416 | lrm: 1.00 | dt: 2496.58ms | tok/sec: 210,002 | bf16_mfu: 32.06 | epoch: 1 pq: 5 rg: 80 | total time: 20.65m | eta: 41.5m +step 00506/01500 (33.73%) | loss: 2.636152 | lrm: 1.00 | dt: 2498.08ms | tok/sec: 209,876 | bf16_mfu: 32.04 | epoch: 1 pq: 5 rg: 80 | total time: 20.69m | eta: 41.5m +step 00507/01500 (33.80%) | loss: 2.624096 | lrm: 1.00 | dt: 2502.37ms | tok/sec: 209,516 | bf16_mfu: 31.99 | epoch: 1 pq: 5 rg: 80 | total time: 20.74m | eta: 41.4m +step 00508/01500 (33.87%) | loss: 2.641861 | lrm: 1.00 | dt: 2499.42ms | tok/sec: 209,763 | bf16_mfu: 32.03 | epoch: 1 pq: 5 rg: 80 | total time: 20.78m | eta: 41.4m +step 00509/01500 (33.93%) | loss: 2.601222 | lrm: 1.00 | dt: 2505.30ms | tok/sec: 209,271 | bf16_mfu: 31.95 | epoch: 1 pq: 5 rg: 80 | total time: 20.82m | eta: 41.3m +step 00510/01500 (34.00%) | loss: 2.568798 | lrm: 1.00 | dt: 2501.78ms | tok/sec: 209,565 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 0 | total time: 20.86m | eta: 41.3m +step 00511/01500 (34.07%) | loss: 2.529884 | lrm: 1.00 | dt: 2493.74ms | tok/sec: 210,241 | bf16_mfu: 32.10 | epoch: 1 pq: 6 rg: 0 | total time: 20.90m | eta: 41.3m +step 00512/01500 (34.13%) | loss: 2.572685 | lrm: 1.00 | dt: 2497.54ms | tok/sec: 209,922 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 0 | total time: 20.94m | eta: 41.2m +step 00513/01500 (34.20%) | loss: 2.588677 | lrm: 1.00 | dt: 2498.74ms | tok/sec: 209,821 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 0 | total time: 20.99m | eta: 41.2m +step 00514/01500 (34.27%) | loss: 2.607854 | lrm: 1.00 | dt: 2497.14ms | tok/sec: 209,955 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 0 | total time: 21.03m | eta: 41.1m +step 00515/01500 (34.33%) | loss: 2.605475 | lrm: 1.00 | dt: 2506.18ms | tok/sec: 209,198 | bf16_mfu: 31.94 | epoch: 1 pq: 6 rg: 0 | total time: 21.07m | eta: 41.1m +step 00516/01500 (34.40%) | loss: 2.625303 | lrm: 1.00 | dt: 2497.45ms | tok/sec: 209,929 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 0 | total time: 21.11m | eta: 41.1m +step 00517/01500 (34.47%) | loss: 2.591898 | lrm: 1.00 | dt: 2498.23ms | tok/sec: 209,864 | bf16_mfu: 32.04 | epoch: 1 pq: 6 rg: 0 | total time: 21.15m | eta: 41.0m +step 00518/01500 (34.53%) | loss: 2.607237 | lrm: 1.00 | dt: 2509.66ms | tok/sec: 208,907 | bf16_mfu: 31.89 | epoch: 1 pq: 6 rg: 0 | total time: 21.19m | eta: 41.0m +step 00519/01500 (34.60%) | loss: 2.591363 | lrm: 1.00 | dt: 2502.36ms | tok/sec: 209,517 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 8 | total time: 21.24m | eta: 40.9m +step 00520/01500 (34.67%) | loss: 2.605786 | lrm: 1.00 | dt: 2501.47ms | tok/sec: 209,592 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 8 | total time: 21.28m | eta: 40.9m +step 00521/01500 (34.73%) | loss: 2.601866 | lrm: 1.00 | dt: 2498.37ms | tok/sec: 209,851 | bf16_mfu: 32.04 | epoch: 1 pq: 6 rg: 8 | total time: 21.32m | eta: 40.8m +step 00522/01500 (34.80%) | loss: 2.550722 | lrm: 1.00 | dt: 2501.55ms | tok/sec: 209,585 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 8 | total time: 21.36m | eta: 40.8m +step 00523/01500 (34.87%) | loss: 2.519845 | lrm: 1.00 | dt: 2502.27ms | tok/sec: 209,524 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 8 | total time: 21.40m | eta: 40.8m +step 00524/01500 (34.93%) | loss: 2.545520 | lrm: 1.00 | dt: 2502.08ms | tok/sec: 209,540 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 8 | total time: 21.44m | eta: 40.7m +step 00525/01500 (35.00%) | loss: 2.555268 | lrm: 1.00 | dt: 2502.19ms | tok/sec: 209,531 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 8 | total time: 21.49m | eta: 40.7m +step 00526/01500 (35.07%) | loss: 2.545308 | lrm: 1.00 | dt: 2495.97ms | tok/sec: 210,053 | bf16_mfu: 32.07 | epoch: 1 pq: 6 rg: 8 | total time: 21.53m | eta: 40.6m +step 00527/01500 (35.13%) | loss: 2.570636 | lrm: 1.00 | dt: 2502.35ms | tok/sec: 209,518 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 16 | total time: 21.57m | eta: 40.6m +step 00528/01500 (35.20%) | loss: 2.579141 | lrm: 1.00 | dt: 2495.69ms | tok/sec: 210,077 | bf16_mfu: 32.07 | epoch: 1 pq: 6 rg: 16 | total time: 21.61m | eta: 40.6m +step 00529/01500 (35.27%) | loss: 2.580171 | lrm: 1.00 | dt: 2498.88ms | tok/sec: 209,808 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 16 | total time: 21.65m | eta: 40.5m +step 00530/01500 (35.33%) | loss: 2.548039 | lrm: 1.00 | dt: 2500.94ms | tok/sec: 209,636 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 16 | total time: 21.69m | eta: 40.5m +step 00531/01500 (35.40%) | loss: 2.502980 | lrm: 1.00 | dt: 2500.77ms | tok/sec: 209,650 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 16 | total time: 21.74m | eta: 40.4m +step 00532/01500 (35.47%) | loss: 2.525493 | lrm: 1.00 | dt: 2501.45ms | tok/sec: 209,594 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 16 | total time: 21.78m | eta: 40.4m +step 00533/01500 (35.53%) | loss: 2.506324 | lrm: 1.00 | dt: 2499.94ms | tok/sec: 209,720 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 16 | total time: 21.82m | eta: 40.3m +step 00534/01500 (35.60%) | loss: 2.501816 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 16 | total time: 21.86m | eta: 40.3m +step 00535/01500 (35.67%) | loss: 2.491864 | lrm: 1.00 | dt: 2504.63ms | tok/sec: 209,327 | bf16_mfu: 31.96 | epoch: 1 pq: 6 rg: 24 | total time: 21.90m | eta: 40.3m +step 00536/01500 (35.73%) | loss: 2.486440 | lrm: 1.00 | dt: 2500.08ms | tok/sec: 209,708 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 24 | total time: 21.94m | eta: 40.2m +step 00537/01500 (35.80%) | loss: 2.434391 | lrm: 1.00 | dt: 2500.46ms | tok/sec: 209,676 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 24 | total time: 21.99m | eta: 40.2m +step 00538/01500 (35.87%) | loss: 2.410299 | lrm: 1.00 | dt: 2500.80ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 24 | total time: 22.03m | eta: 40.1m +step 00539/01500 (35.93%) | loss: 2.355827 | lrm: 1.00 | dt: 2500.74ms | tok/sec: 209,652 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 24 | total time: 22.07m | eta: 40.1m +step 00540/01500 (36.00%) | loss: 2.366697 | lrm: 1.00 | dt: 2501.53ms | tok/sec: 209,586 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 24 | total time: 22.11m | eta: 40.0m +step 00541/01500 (36.07%) | loss: 2.375723 | lrm: 1.00 | dt: 3375.97ms | tok/sec: 155,299 | bf16_mfu: 23.71 | epoch: 1 pq: 6 rg: 24 | total time: 22.17m | eta: 40.0m +step 00542/01500 (36.13%) | loss: 2.350408 | lrm: 1.00 | dt: 2494.44ms | tok/sec: 210,182 | bf16_mfu: 32.09 | epoch: 1 pq: 6 rg: 24 | total time: 22.21m | eta: 40.0m +step 00543/01500 (36.20%) | loss: 2.399915 | lrm: 1.00 | dt: 2495.23ms | tok/sec: 210,116 | bf16_mfu: 32.08 | epoch: 1 pq: 6 rg: 32 | total time: 22.25m | eta: 40.0m +step 00544/01500 (36.27%) | loss: 2.404802 | lrm: 1.00 | dt: 2497.91ms | tok/sec: 209,890 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 32 | total time: 22.29m | eta: 39.9m +step 00545/01500 (36.33%) | loss: 2.411226 | lrm: 1.00 | dt: 2502.94ms | tok/sec: 209,469 | bf16_mfu: 31.98 | epoch: 1 pq: 6 rg: 32 | total time: 22.33m | eta: 39.9m +step 00546/01500 (36.40%) | loss: 2.450549 | lrm: 1.00 | dt: 2499.50ms | tok/sec: 209,757 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 32 | total time: 22.38m | eta: 39.8m +step 00547/01500 (36.47%) | loss: 2.512951 | lrm: 1.00 | dt: 2501.35ms | tok/sec: 209,601 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 32 | total time: 22.42m | eta: 39.8m +step 00548/01500 (36.53%) | loss: 2.452717 | lrm: 1.00 | dt: 2497.34ms | tok/sec: 209,938 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 32 | total time: 22.46m | eta: 39.7m +step 00549/01500 (36.60%) | loss: 2.430202 | lrm: 1.00 | dt: 2501.50ms | tok/sec: 209,589 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 32 | total time: 22.50m | eta: 39.7m +step 00550/01500 (36.67%) | loss: 2.483732 | lrm: 1.00 | dt: 2498.78ms | tok/sec: 209,817 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 32 | total time: 22.54m | eta: 39.7m +step 00551/01500 (36.73%) | loss: 2.436023 | lrm: 1.00 | dt: 2502.61ms | tok/sec: 209,496 | bf16_mfu: 31.98 | epoch: 1 pq: 6 rg: 40 | total time: 22.58m | eta: 39.6m +step 00552/01500 (36.80%) | loss: 2.370890 | lrm: 1.00 | dt: 2501.94ms | tok/sec: 209,552 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 40 | total time: 22.63m | eta: 39.6m +step 00553/01500 (36.87%) | loss: 2.398080 | lrm: 1.00 | dt: 2499.48ms | tok/sec: 209,759 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 40 | total time: 22.67m | eta: 39.5m +step 00554/01500 (36.93%) | loss: 2.441161 | lrm: 1.00 | dt: 2499.22ms | tok/sec: 209,780 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 40 | total time: 22.71m | eta: 39.5m +step 00555/01500 (37.00%) | loss: 2.442970 | lrm: 1.00 | dt: 2515.28ms | tok/sec: 208,440 | bf16_mfu: 31.82 | epoch: 1 pq: 6 rg: 40 | total time: 22.75m | eta: 39.4m +step 00556/01500 (37.07%) | loss: 2.426135 | lrm: 1.00 | dt: 2497.45ms | tok/sec: 209,928 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 40 | total time: 22.79m | eta: 39.4m +step 00557/01500 (37.13%) | loss: 2.446479 | lrm: 1.00 | dt: 2499.78ms | tok/sec: 209,733 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 40 | total time: 22.83m | eta: 39.4m +step 00558/01500 (37.20%) | loss: 2.498584 | lrm: 1.00 | dt: 2495.45ms | tok/sec: 210,097 | bf16_mfu: 32.08 | epoch: 1 pq: 6 rg: 40 | total time: 22.88m | eta: 39.3m +step 00559/01500 (37.27%) | loss: 2.485985 | lrm: 1.00 | dt: 2502.52ms | tok/sec: 209,503 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 48 | total time: 22.92m | eta: 39.3m +step 00560/01500 (37.33%) | loss: 2.494606 | lrm: 1.00 | dt: 2500.85ms | tok/sec: 209,644 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 48 | total time: 22.96m | eta: 39.2m +step 00561/01500 (37.40%) | loss: 2.515181 | lrm: 1.00 | dt: 2504.71ms | tok/sec: 209,321 | bf16_mfu: 31.96 | epoch: 1 pq: 6 rg: 48 | total time: 23.00m | eta: 39.2m +step 00562/01500 (37.47%) | loss: 2.476007 | lrm: 1.00 | dt: 2500.38ms | tok/sec: 209,682 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 48 | total time: 23.04m | eta: 39.2m +step 00563/01500 (37.53%) | loss: 2.465159 | lrm: 1.00 | dt: 2500.36ms | tok/sec: 209,685 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 48 | total time: 23.08m | eta: 39.1m +step 00564/01500 (37.60%) | loss: 2.490591 | lrm: 1.00 | dt: 2500.24ms | tok/sec: 209,695 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 48 | total time: 23.13m | eta: 39.1m +step 00565/01500 (37.67%) | loss: 2.511817 | lrm: 1.00 | dt: 2498.87ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 48 | total time: 23.17m | eta: 39.0m +step 00566/01500 (37.73%) | loss: 2.482246 | lrm: 1.00 | dt: 2497.28ms | tok/sec: 209,943 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 48 | total time: 23.21m | eta: 39.0m +step 00567/01500 (37.80%) | loss: 2.484301 | lrm: 1.00 | dt: 2500.44ms | tok/sec: 209,678 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 56 | total time: 23.25m | eta: 38.9m +step 00568/01500 (37.87%) | loss: 2.439863 | lrm: 1.00 | dt: 2499.93ms | tok/sec: 209,720 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 56 | total time: 23.29m | eta: 38.9m +step 00569/01500 (37.93%) | loss: 2.414052 | lrm: 1.00 | dt: 2504.69ms | tok/sec: 209,322 | bf16_mfu: 31.96 | epoch: 1 pq: 6 rg: 56 | total time: 23.33m | eta: 38.9m +step 00570/01500 (38.00%) | loss: 2.429948 | lrm: 1.00 | dt: 2497.33ms | tok/sec: 209,939 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 56 | total time: 23.38m | eta: 38.8m +step 00571/01500 (38.07%) | loss: 2.440057 | lrm: 1.00 | dt: 2500.07ms | tok/sec: 209,709 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 56 | total time: 23.42m | eta: 38.8m +step 00572/01500 (38.13%) | loss: 2.492334 | lrm: 1.00 | dt: 2498.79ms | tok/sec: 209,817 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 56 | total time: 23.46m | eta: 38.7m +step 00573/01500 (38.20%) | loss: 2.540238 | lrm: 1.00 | dt: 2500.43ms | tok/sec: 209,679 | bf16_mfu: 32.01 | epoch: 1 pq: 6 rg: 56 | total time: 23.50m | eta: 38.7m +step 00574/01500 (38.27%) | loss: 2.579401 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,740 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 56 | total time: 23.54m | eta: 38.7m +step 00575/01500 (38.33%) | loss: 2.539988 | lrm: 1.00 | dt: 2497.77ms | tok/sec: 209,902 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 64 | total time: 23.58m | eta: 38.6m +step 00576/01500 (38.40%) | loss: 2.479109 | lrm: 1.00 | dt: 2501.94ms | tok/sec: 209,552 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 64 | total time: 23.63m | eta: 38.6m +step 00577/01500 (38.47%) | loss: 2.549384 | lrm: 1.00 | dt: 2506.03ms | tok/sec: 209,210 | bf16_mfu: 31.94 | epoch: 1 pq: 6 rg: 64 | total time: 23.67m | eta: 38.5m +step 00578/01500 (38.53%) | loss: 2.503151 | lrm: 1.00 | dt: 2496.43ms | tok/sec: 210,015 | bf16_mfu: 32.06 | epoch: 1 pq: 6 rg: 64 | total time: 23.71m | eta: 38.5m +step 00579/01500 (38.60%) | loss: 2.483110 | lrm: 1.00 | dt: 2503.21ms | tok/sec: 209,446 | bf16_mfu: 31.98 | epoch: 1 pq: 6 rg: 64 | total time: 23.75m | eta: 38.4m +step 00580/01500 (38.67%) | loss: 2.403492 | lrm: 1.00 | dt: 2499.16ms | tok/sec: 209,785 | bf16_mfu: 32.03 | epoch: 1 pq: 6 rg: 64 | total time: 23.79m | eta: 38.4m +step 00581/01500 (38.73%) | loss: 2.474327 | lrm: 1.00 | dt: 2498.07ms | tok/sec: 209,877 | bf16_mfu: 32.04 | epoch: 1 pq: 6 rg: 64 | total time: 23.83m | eta: 38.4m +step 00582/01500 (38.80%) | loss: 2.492671 | lrm: 1.00 | dt: 2500.23ms | tok/sec: 209,696 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 64 | total time: 23.88m | eta: 38.3m +step 00583/01500 (38.87%) | loss: 2.491891 | lrm: 1.00 | dt: 2502.13ms | tok/sec: 209,536 | bf16_mfu: 31.99 | epoch: 1 pq: 6 rg: 72 | total time: 23.92m | eta: 38.3m +step 00584/01500 (38.93%) | loss: 2.457806 | lrm: 1.00 | dt: 2497.86ms | tok/sec: 209,894 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 72 | total time: 23.96m | eta: 38.2m +step 00585/01500 (39.00%) | loss: 2.450195 | lrm: 1.00 | dt: 2506.96ms | tok/sec: 209,132 | bf16_mfu: 31.93 | epoch: 1 pq: 6 rg: 72 | total time: 24.00m | eta: 38.2m +step 00586/01500 (39.07%) | loss: 2.478761 | lrm: 1.00 | dt: 2497.61ms | tok/sec: 209,915 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 72 | total time: 24.04m | eta: 38.2m +step 00587/01500 (39.13%) | loss: 2.517311 | lrm: 1.00 | dt: 2500.22ms | tok/sec: 209,696 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 72 | total time: 24.08m | eta: 38.1m +step 00588/01500 (39.20%) | loss: 2.535973 | lrm: 1.00 | dt: 2503.20ms | tok/sec: 209,446 | bf16_mfu: 31.98 | epoch: 1 pq: 6 rg: 72 | total time: 24.13m | eta: 38.1m +step 00589/01500 (39.27%) | loss: 2.500935 | lrm: 1.00 | dt: 2501.28ms | tok/sec: 209,608 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 72 | total time: 24.17m | eta: 38.0m +step 00590/01500 (39.33%) | loss: 2.467282 | lrm: 1.00 | dt: 2497.19ms | tok/sec: 209,951 | bf16_mfu: 32.05 | epoch: 1 pq: 6 rg: 72 | total time: 24.21m | eta: 38.0m +step 00591/01500 (39.40%) | loss: 2.462404 | lrm: 1.00 | dt: 2495.94ms | tok/sec: 210,056 | bf16_mfu: 32.07 | epoch: 1 pq: 6 rg: 72 | total time: 24.25m | eta: 37.9m +step 00592/01500 (39.47%) | loss: 2.442026 | lrm: 1.00 | dt: 2499.57ms | tok/sec: 209,751 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 80 | total time: 24.29m | eta: 37.9m +step 00593/01500 (39.53%) | loss: 2.445343 | lrm: 1.00 | dt: 2499.51ms | tok/sec: 209,756 | bf16_mfu: 32.02 | epoch: 1 pq: 6 rg: 80 | total time: 24.33m | eta: 37.9m +step 00594/01500 (39.60%) | loss: 2.444952 | lrm: 1.00 | dt: 2501.17ms | tok/sec: 209,616 | bf16_mfu: 32.00 | epoch: 1 pq: 6 rg: 80 | total time: 24.38m | eta: 37.8m +step 00595/01500 (39.67%) | loss: 2.503910 | lrm: 1.00 | dt: 2505.30ms | tok/sec: 209,271 | bf16_mfu: 31.95 | epoch: 1 pq: 6 rg: 80 | total time: 24.42m | eta: 37.8m +step 00596/01500 (39.73%) | loss: 2.517616 | lrm: 1.00 | dt: 2495.79ms | tok/sec: 210,069 | bf16_mfu: 32.07 | epoch: 1 pq: 6 rg: 80 | total time: 24.46m | eta: 37.7m +step 00597/01500 (39.80%) | loss: 2.523563 | lrm: 1.00 | dt: 2504.03ms | tok/sec: 209,377 | bf16_mfu: 31.97 | epoch: 1 pq: 6 rg: 80 | total time: 24.50m | eta: 37.7m +step 00598/01500 (39.87%) | loss: 2.520262 | lrm: 1.00 | dt: 2498.34ms | tok/sec: 209,854 | bf16_mfu: 32.04 | epoch: 1 pq: 6 rg: 80 | total time: 24.54m | eta: 37.6m +step 00599/01500 (39.93%) | loss: 2.546318 | lrm: 1.00 | dt: 2503.60ms | tok/sec: 209,413 | bf16_mfu: 31.97 | epoch: 1 pq: 6 rg: 80 | total time: 24.58m | eta: 37.6m +Step 00600 | Validation bpb: 0.751316 +Step 00600 | ICL early: 2.7678 late: 2.9164 score: +0.1486 | Induction loss: 10.1360 acc: 0.0637 (random-half: 13.9930) +step 00600/01500 (40.00%) | loss: 2.573231 | lrm: 1.00 | dt: 2495.06ms | tok/sec: 210,130 | bf16_mfu: 32.08 | epoch: 1 pq: 7 rg: 0 | total time: 24.63m | eta: 37.6m +step 00601/01500 (40.07%) | loss: 2.590959 | lrm: 1.00 | dt: 2499.26ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 0 | total time: 24.67m | eta: 37.5m +step 00602/01500 (40.13%) | loss: 2.623053 | lrm: 1.00 | dt: 2497.92ms | tok/sec: 209,890 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 0 | total time: 24.71m | eta: 37.5m +step 00603/01500 (40.20%) | loss: 2.657384 | lrm: 1.00 | dt: 2497.67ms | tok/sec: 209,911 | bf16_mfu: 32.05 | epoch: 1 pq: 7 rg: 0 | total time: 24.75m | eta: 37.4m +step 00604/01500 (40.27%) | loss: 2.634753 | lrm: 1.00 | dt: 2502.15ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 0 | total time: 24.79m | eta: 37.4m +step 00605/01500 (40.33%) | loss: 2.635326 | lrm: 1.00 | dt: 2498.93ms | tok/sec: 209,804 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 0 | total time: 24.83m | eta: 37.4m +step 00606/01500 (40.40%) | loss: 2.613039 | lrm: 1.00 | dt: 2503.49ms | tok/sec: 209,422 | bf16_mfu: 31.97 | epoch: 1 pq: 7 rg: 0 | total time: 24.88m | eta: 37.3m +step 00607/01500 (40.47%) | loss: 2.566059 | lrm: 1.00 | dt: 2497.07ms | tok/sec: 209,961 | bf16_mfu: 32.06 | epoch: 1 pq: 7 rg: 0 | total time: 24.92m | eta: 37.3m +step 00608/01500 (40.53%) | loss: 2.564715 | lrm: 1.00 | dt: 2498.68ms | tok/sec: 209,825 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 8 | total time: 24.96m | eta: 37.2m +step 00609/01500 (40.60%) | loss: 2.534109 | lrm: 1.00 | dt: 2499.28ms | tok/sec: 209,775 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 8 | total time: 25.00m | eta: 37.2m +step 00610/01500 (40.67%) | loss: 2.484601 | lrm: 1.00 | dt: 2500.25ms | tok/sec: 209,694 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 8 | total time: 25.04m | eta: 37.1m +step 00611/01500 (40.73%) | loss: 2.481609 | lrm: 1.00 | dt: 2499.37ms | tok/sec: 209,767 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 8 | total time: 25.08m | eta: 37.1m +step 00612/01500 (40.80%) | loss: 2.523672 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 8 | total time: 25.13m | eta: 37.1m +step 00613/01500 (40.87%) | loss: 2.490175 | lrm: 1.00 | dt: 2502.44ms | tok/sec: 209,510 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 8 | total time: 25.17m | eta: 37.0m +step 00614/01500 (40.93%) | loss: 2.499527 | lrm: 1.00 | dt: 2502.15ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 8 | total time: 25.21m | eta: 37.0m +step 00615/01500 (41.00%) | loss: 2.466889 | lrm: 1.00 | dt: 2503.91ms | tok/sec: 209,387 | bf16_mfu: 31.97 | epoch: 1 pq: 7 rg: 8 | total time: 25.25m | eta: 36.9m +step 00616/01500 (41.07%) | loss: 2.544195 | lrm: 1.00 | dt: 2503.06ms | tok/sec: 209,459 | bf16_mfu: 31.98 | epoch: 1 pq: 7 rg: 16 | total time: 25.29m | eta: 36.9m +step 00617/01500 (41.13%) | loss: 2.536221 | lrm: 1.00 | dt: 2499.84ms | tok/sec: 209,728 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 16 | total time: 25.33m | eta: 36.9m +step 00618/01500 (41.20%) | loss: 2.591457 | lrm: 1.00 | dt: 2498.25ms | tok/sec: 209,861 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 16 | total time: 25.38m | eta: 36.8m +step 00619/01500 (41.27%) | loss: 2.605541 | lrm: 1.00 | dt: 2502.96ms | tok/sec: 209,467 | bf16_mfu: 31.98 | epoch: 1 pq: 7 rg: 16 | total time: 25.42m | eta: 36.8m +step 00620/01500 (41.33%) | loss: 2.558934 | lrm: 1.00 | dt: 2501.61ms | tok/sec: 209,580 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 16 | total time: 25.46m | eta: 36.7m +step 00621/01500 (41.40%) | loss: 2.563063 | lrm: 1.00 | dt: 2645.23ms | tok/sec: 198,201 | bf16_mfu: 30.26 | epoch: 1 pq: 7 rg: 16 | total time: 25.50m | eta: 36.7m +step 00622/01500 (41.47%) | loss: 2.549821 | lrm: 1.00 | dt: 2501.61ms | tok/sec: 209,580 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 16 | total time: 25.54m | eta: 36.6m +step 00623/01500 (41.53%) | loss: 2.532256 | lrm: 1.00 | dt: 2500.50ms | tok/sec: 209,673 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 16 | total time: 25.59m | eta: 36.6m +step 00624/01500 (41.60%) | loss: 2.536579 | lrm: 1.00 | dt: 2506.06ms | tok/sec: 209,207 | bf16_mfu: 31.94 | epoch: 1 pq: 7 rg: 24 | total time: 25.63m | eta: 36.6m +step 00625/01500 (41.67%) | loss: 2.582130 | lrm: 1.00 | dt: 2497.51ms | tok/sec: 209,924 | bf16_mfu: 32.05 | epoch: 1 pq: 7 rg: 24 | total time: 25.67m | eta: 36.5m +step 00626/01500 (41.73%) | loss: 2.585435 | lrm: 1.00 | dt: 2497.79ms | tok/sec: 209,900 | bf16_mfu: 32.05 | epoch: 1 pq: 7 rg: 24 | total time: 25.71m | eta: 36.5m +step 00627/01500 (41.80%) | loss: 2.578101 | lrm: 1.00 | dt: 2499.72ms | tok/sec: 209,739 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 24 | total time: 25.75m | eta: 36.4m +step 00628/01500 (41.87%) | loss: 2.586965 | lrm: 1.00 | dt: 2499.31ms | tok/sec: 209,773 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 24 | total time: 25.79m | eta: 36.4m +step 00629/01500 (41.93%) | loss: 2.558031 | lrm: 1.00 | dt: 2503.92ms | tok/sec: 209,386 | bf16_mfu: 31.97 | epoch: 1 pq: 7 rg: 24 | total time: 25.84m | eta: 36.4m +step 00630/01500 (42.00%) | loss: 2.555793 | lrm: 1.00 | dt: 2499.98ms | tok/sec: 209,717 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 24 | total time: 25.88m | eta: 36.3m +step 00631/01500 (42.07%) | loss: 2.562231 | lrm: 1.00 | dt: 2503.98ms | tok/sec: 209,382 | bf16_mfu: 31.97 | epoch: 1 pq: 7 rg: 24 | total time: 25.92m | eta: 36.3m +step 00632/01500 (42.13%) | loss: 2.553953 | lrm: 1.00 | dt: 2498.23ms | tok/sec: 209,863 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 24 | total time: 25.96m | eta: 36.2m +step 00633/01500 (42.20%) | loss: 2.562541 | lrm: 1.00 | dt: 2496.87ms | tok/sec: 209,978 | bf16_mfu: 32.06 | epoch: 1 pq: 7 rg: 32 | total time: 26.00m | eta: 36.2m +step 00634/01500 (42.27%) | loss: 2.567383 | lrm: 1.00 | dt: 2499.62ms | tok/sec: 209,747 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 32 | total time: 26.05m | eta: 36.1m +step 00635/01500 (42.33%) | loss: 2.529419 | lrm: 1.00 | dt: 2504.54ms | tok/sec: 209,334 | bf16_mfu: 31.96 | epoch: 1 pq: 7 rg: 32 | total time: 26.09m | eta: 36.1m +step 00636/01500 (42.40%) | loss: 2.557127 | lrm: 1.00 | dt: 2500.53ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 32 | total time: 26.13m | eta: 36.1m +step 00637/01500 (42.47%) | loss: 2.555620 | lrm: 1.00 | dt: 2499.21ms | tok/sec: 209,781 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 32 | total time: 26.17m | eta: 36.0m +step 00638/01500 (42.53%) | loss: 2.502341 | lrm: 1.00 | dt: 2498.16ms | tok/sec: 209,869 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 32 | total time: 26.21m | eta: 36.0m +step 00639/01500 (42.60%) | loss: 2.480235 | lrm: 1.00 | dt: 2497.12ms | tok/sec: 209,957 | bf16_mfu: 32.06 | epoch: 1 pq: 7 rg: 32 | total time: 26.25m | eta: 35.9m +step 00640/01500 (42.67%) | loss: 2.470096 | lrm: 1.00 | dt: 2499.12ms | tok/sec: 209,788 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 32 | total time: 26.30m | eta: 35.9m +step 00641/01500 (42.73%) | loss: 2.460140 | lrm: 1.00 | dt: 2494.71ms | tok/sec: 210,159 | bf16_mfu: 32.09 | epoch: 1 pq: 7 rg: 40 | total time: 26.34m | eta: 35.9m +step 00642/01500 (42.80%) | loss: 2.454784 | lrm: 1.00 | dt: 2506.16ms | tok/sec: 209,199 | bf16_mfu: 31.94 | epoch: 1 pq: 7 rg: 40 | total time: 26.38m | eta: 35.8m +step 00643/01500 (42.87%) | loss: 2.453636 | lrm: 1.00 | dt: 2504.26ms | tok/sec: 209,358 | bf16_mfu: 31.96 | epoch: 1 pq: 7 rg: 40 | total time: 26.42m | eta: 35.8m +step 00644/01500 (42.93%) | loss: 2.493934 | lrm: 1.00 | dt: 2498.31ms | tok/sec: 209,857 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 40 | total time: 26.46m | eta: 35.7m +step 00645/01500 (43.00%) | loss: 2.482507 | lrm: 1.00 | dt: 2497.23ms | tok/sec: 209,948 | bf16_mfu: 32.05 | epoch: 1 pq: 7 rg: 40 | total time: 26.50m | eta: 35.7m +step 00646/01500 (43.07%) | loss: 2.500603 | lrm: 1.00 | dt: 2501.29ms | tok/sec: 209,607 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 40 | total time: 26.55m | eta: 35.6m +step 00647/01500 (43.13%) | loss: 2.483368 | lrm: 1.00 | dt: 2503.37ms | tok/sec: 209,432 | bf16_mfu: 31.98 | epoch: 1 pq: 7 rg: 40 | total time: 26.59m | eta: 35.6m +step 00648/01500 (43.20%) | loss: 2.496198 | lrm: 1.00 | dt: 2500.98ms | tok/sec: 209,632 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 40 | total time: 26.63m | eta: 35.6m +step 00649/01500 (43.27%) | loss: 2.470514 | lrm: 1.00 | dt: 2499.53ms | tok/sec: 209,754 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 48 | total time: 26.67m | eta: 35.5m +step 00650/01500 (43.33%) | loss: 2.483934 | lrm: 1.00 | dt: 2501.27ms | tok/sec: 209,608 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 48 | total time: 26.71m | eta: 35.5m +step 00651/01500 (43.40%) | loss: 2.463478 | lrm: 1.00 | dt: 2505.83ms | tok/sec: 209,227 | bf16_mfu: 31.94 | epoch: 1 pq: 7 rg: 48 | total time: 26.75m | eta: 35.4m +step 00652/01500 (43.47%) | loss: 2.462173 | lrm: 1.00 | dt: 2504.54ms | tok/sec: 209,335 | bf16_mfu: 31.96 | epoch: 1 pq: 7 rg: 48 | total time: 26.80m | eta: 35.4m +step 00653/01500 (43.53%) | loss: 2.452519 | lrm: 1.00 | dt: 2499.08ms | tok/sec: 209,792 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 48 | total time: 26.84m | eta: 35.4m +step 00654/01500 (43.60%) | loss: 2.466888 | lrm: 1.00 | dt: 2501.52ms | tok/sec: 209,587 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 48 | total time: 26.88m | eta: 35.3m +step 00655/01500 (43.67%) | loss: 2.389489 | lrm: 1.00 | dt: 2502.10ms | tok/sec: 209,539 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 48 | total time: 26.92m | eta: 35.3m +step 00656/01500 (43.73%) | loss: 2.398192 | lrm: 1.00 | dt: 2500.36ms | tok/sec: 209,685 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 48 | total time: 26.96m | eta: 35.2m +step 00657/01500 (43.80%) | loss: 2.424800 | lrm: 1.00 | dt: 2498.61ms | tok/sec: 209,832 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 56 | total time: 27.00m | eta: 35.2m +step 00658/01500 (43.87%) | loss: 2.439903 | lrm: 1.00 | dt: 2501.29ms | tok/sec: 209,606 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 56 | total time: 27.05m | eta: 35.1m +step 00659/01500 (43.93%) | loss: 2.458521 | lrm: 1.00 | dt: 2500.78ms | tok/sec: 209,649 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 56 | total time: 27.09m | eta: 35.1m +step 00660/01500 (44.00%) | loss: 2.462703 | lrm: 1.00 | dt: 2509.84ms | tok/sec: 208,892 | bf16_mfu: 31.89 | epoch: 1 pq: 7 rg: 56 | total time: 27.13m | eta: 35.1m +step 00661/01500 (44.07%) | loss: 2.472215 | lrm: 1.00 | dt: 2502.80ms | tok/sec: 209,480 | bf16_mfu: 31.98 | epoch: 1 pq: 7 rg: 56 | total time: 27.17m | eta: 35.0m +step 00662/01500 (44.13%) | loss: 2.471677 | lrm: 1.00 | dt: 2504.38ms | tok/sec: 209,348 | bf16_mfu: 31.96 | epoch: 1 pq: 7 rg: 56 | total time: 27.21m | eta: 35.0m +step 00663/01500 (44.20%) | loss: 2.462579 | lrm: 1.00 | dt: 2502.03ms | tok/sec: 209,544 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 56 | total time: 27.25m | eta: 34.9m +step 00664/01500 (44.27%) | loss: 2.475935 | lrm: 1.00 | dt: 2501.32ms | tok/sec: 209,604 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 56 | total time: 27.30m | eta: 34.9m +step 00665/01500 (44.33%) | loss: 2.480156 | lrm: 1.00 | dt: 2495.91ms | tok/sec: 210,058 | bf16_mfu: 32.07 | epoch: 1 pq: 7 rg: 64 | total time: 27.34m | eta: 34.8m +step 00666/01500 (44.40%) | loss: 2.508779 | lrm: 1.00 | dt: 2496.96ms | tok/sec: 209,970 | bf16_mfu: 32.06 | epoch: 1 pq: 7 rg: 64 | total time: 27.38m | eta: 34.8m +step 00667/01500 (44.47%) | loss: 2.471590 | lrm: 1.00 | dt: 2506.19ms | tok/sec: 209,197 | bf16_mfu: 31.94 | epoch: 1 pq: 7 rg: 64 | total time: 27.42m | eta: 34.8m +step 00668/01500 (44.53%) | loss: 2.491536 | lrm: 1.00 | dt: 2499.73ms | tok/sec: 209,738 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 64 | total time: 27.46m | eta: 34.7m +step 00669/01500 (44.60%) | loss: 2.513735 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 64 | total time: 27.50m | eta: 34.7m +step 00670/01500 (44.67%) | loss: 2.491293 | lrm: 1.00 | dt: 2498.09ms | tok/sec: 209,875 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 64 | total time: 27.55m | eta: 34.6m +step 00671/01500 (44.73%) | loss: 2.493976 | lrm: 1.00 | dt: 2495.46ms | tok/sec: 210,096 | bf16_mfu: 32.08 | epoch: 1 pq: 7 rg: 64 | total time: 27.59m | eta: 34.6m +step 00672/01500 (44.80%) | loss: 2.483517 | lrm: 1.00 | dt: 2496.53ms | tok/sec: 210,006 | bf16_mfu: 32.06 | epoch: 1 pq: 7 rg: 64 | total time: 27.63m | eta: 34.6m +step 00673/01500 (44.87%) | loss: 2.500287 | lrm: 1.00 | dt: 2499.84ms | tok/sec: 209,728 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 72 | total time: 27.67m | eta: 34.5m +step 00674/01500 (44.93%) | loss: 2.473263 | lrm: 1.00 | dt: 2500.21ms | tok/sec: 209,697 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 72 | total time: 27.71m | eta: 34.5m +step 00675/01500 (45.00%) | loss: 2.487736 | lrm: 1.00 | dt: 2500.47ms | tok/sec: 209,675 | bf16_mfu: 32.01 | epoch: 1 pq: 7 rg: 72 | total time: 27.75m | eta: 34.4m +step 00676/01500 (45.07%) | loss: 2.477532 | lrm: 1.00 | dt: 2504.80ms | tok/sec: 209,313 | bf16_mfu: 31.96 | epoch: 1 pq: 7 rg: 72 | total time: 27.80m | eta: 34.4m +step 00677/01500 (45.13%) | loss: 2.454688 | lrm: 1.00 | dt: 2499.50ms | tok/sec: 209,757 | bf16_mfu: 32.02 | epoch: 1 pq: 7 rg: 72 | total time: 27.84m | eta: 34.3m +step 00678/01500 (45.20%) | loss: 2.430004 | lrm: 1.00 | dt: 2498.98ms | tok/sec: 209,800 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 72 | total time: 27.88m | eta: 34.3m +step 00679/01500 (45.27%) | loss: 2.466982 | lrm: 1.00 | dt: 2505.25ms | tok/sec: 209,276 | bf16_mfu: 31.95 | epoch: 1 pq: 7 rg: 72 | total time: 27.92m | eta: 34.3m +step 00680/01500 (45.33%) | loss: 2.479966 | lrm: 1.00 | dt: 2495.83ms | tok/sec: 210,066 | bf16_mfu: 32.07 | epoch: 1 pq: 7 rg: 72 | total time: 27.96m | eta: 34.2m +step 00681/01500 (45.40%) | loss: 2.485721 | lrm: 1.00 | dt: 2498.94ms | tok/sec: 209,804 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 80 | total time: 28.00m | eta: 34.2m +step 00682/01500 (45.47%) | loss: 2.496000 | lrm: 1.00 | dt: 2501.68ms | tok/sec: 209,574 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 80 | total time: 28.05m | eta: 34.1m +step 00683/01500 (45.53%) | loss: 2.475892 | lrm: 1.00 | dt: 2499.28ms | tok/sec: 209,775 | bf16_mfu: 32.03 | epoch: 1 pq: 7 rg: 80 | total time: 28.09m | eta: 34.1m +step 00684/01500 (45.60%) | loss: 2.461710 | lrm: 1.00 | dt: 2503.92ms | tok/sec: 209,386 | bf16_mfu: 31.97 | epoch: 1 pq: 7 rg: 80 | total time: 28.13m | eta: 34.1m +step 00685/01500 (45.67%) | loss: 2.436354 | lrm: 1.00 | dt: 2495.77ms | tok/sec: 210,070 | bf16_mfu: 32.07 | epoch: 1 pq: 7 rg: 80 | total time: 28.17m | eta: 34.0m +step 00686/01500 (45.73%) | loss: 2.415804 | lrm: 1.00 | dt: 2501.18ms | tok/sec: 209,616 | bf16_mfu: 32.00 | epoch: 1 pq: 7 rg: 80 | total time: 28.21m | eta: 34.0m +step 00687/01500 (45.80%) | loss: 2.399857 | lrm: 1.00 | dt: 2498.45ms | tok/sec: 209,845 | bf16_mfu: 32.04 | epoch: 1 pq: 7 rg: 80 | total time: 28.25m | eta: 33.9m +step 00688/01500 (45.87%) | loss: 2.421625 | lrm: 1.00 | dt: 2502.31ms | tok/sec: 209,521 | bf16_mfu: 31.99 | epoch: 1 pq: 7 rg: 80 | total time: 28.30m | eta: 33.9m +step 00689/01500 (45.93%) | loss: 2.439345 | lrm: 1.00 | dt: 2501.34ms | tok/sec: 209,602 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 0 | total time: 28.34m | eta: 33.8m +step 00690/01500 (46.00%) | loss: 2.447331 | lrm: 1.00 | dt: 2498.95ms | tok/sec: 209,803 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 0 | total time: 28.38m | eta: 33.8m +step 00691/01500 (46.07%) | loss: 2.425838 | lrm: 1.00 | dt: 2500.67ms | tok/sec: 209,658 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 0 | total time: 28.42m | eta: 33.8m +step 00692/01500 (46.13%) | loss: 2.469981 | lrm: 1.00 | dt: 2496.11ms | tok/sec: 210,041 | bf16_mfu: 32.07 | epoch: 1 pq: 8 rg: 0 | total time: 28.46m | eta: 33.7m +step 00693/01500 (46.20%) | loss: 2.510274 | lrm: 1.00 | dt: 2501.90ms | tok/sec: 209,555 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 0 | total time: 28.50m | eta: 33.7m +step 00694/01500 (46.27%) | loss: 2.501991 | lrm: 1.00 | dt: 2503.50ms | tok/sec: 209,422 | bf16_mfu: 31.97 | epoch: 1 pq: 8 rg: 0 | total time: 28.55m | eta: 33.6m +step 00695/01500 (46.33%) | loss: 2.533423 | lrm: 1.00 | dt: 2502.78ms | tok/sec: 209,482 | bf16_mfu: 31.98 | epoch: 1 pq: 8 rg: 0 | total time: 28.59m | eta: 33.6m +step 00696/01500 (46.40%) | loss: 2.516100 | lrm: 1.00 | dt: 2501.67ms | tok/sec: 209,575 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 8 | total time: 28.63m | eta: 33.6m +step 00697/01500 (46.47%) | loss: 2.490330 | lrm: 1.00 | dt: 2502.44ms | tok/sec: 209,510 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 8 | total time: 28.67m | eta: 33.5m +step 00698/01500 (46.53%) | loss: 2.467871 | lrm: 1.00 | dt: 2501.35ms | tok/sec: 209,601 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 8 | total time: 28.71m | eta: 33.5m +step 00699/01500 (46.60%) | loss: 2.510379 | lrm: 1.00 | dt: 2501.73ms | tok/sec: 209,570 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 8 | total time: 28.75m | eta: 33.4m +Step 00700 | Validation bpb: 0.745837 +Step 00700 | ICL early: 2.7512 late: 2.9013 score: +0.1501 | Induction loss: 9.7585 acc: 0.0784 (random-half: 13.9678) +step 00700/01500 (46.67%) | loss: 2.376646 | lrm: 1.00 | dt: 2498.71ms | tok/sec: 209,823 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 8 | total time: 28.80m | eta: 33.4m +step 00701/01500 (46.73%) | loss: 2.384216 | lrm: 1.00 | dt: 2501.50ms | tok/sec: 209,589 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 8 | total time: 28.84m | eta: 33.3m +step 00702/01500 (46.80%) | loss: 2.357306 | lrm: 1.00 | dt: 2856.88ms | tok/sec: 183,517 | bf16_mfu: 28.02 | epoch: 1 pq: 8 rg: 8 | total time: 28.89m | eta: 33.3m +step 00703/01500 (46.87%) | loss: 2.364936 | lrm: 1.00 | dt: 2500.41ms | tok/sec: 209,680 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 8 | total time: 28.93m | eta: 33.3m +step 00704/01500 (46.93%) | loss: 2.426741 | lrm: 1.00 | dt: 2499.82ms | tok/sec: 209,730 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 8 | total time: 28.97m | eta: 33.2m +step 00705/01500 (47.00%) | loss: 2.429270 | lrm: 1.00 | dt: 2499.99ms | tok/sec: 209,716 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 16 | total time: 29.01m | eta: 33.2m +step 00706/01500 (47.07%) | loss: 2.431152 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,726 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 16 | total time: 29.05m | eta: 33.1m +step 00707/01500 (47.13%) | loss: 2.440887 | lrm: 1.00 | dt: 2501.34ms | tok/sec: 209,603 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 16 | total time: 29.09m | eta: 33.1m +step 00708/01500 (47.20%) | loss: 2.427637 | lrm: 1.00 | dt: 2502.40ms | tok/sec: 209,514 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 16 | total time: 29.14m | eta: 33.1m +step 00709/01500 (47.27%) | loss: 2.426681 | lrm: 1.00 | dt: 2507.04ms | tok/sec: 209,125 | bf16_mfu: 31.93 | epoch: 1 pq: 8 rg: 16 | total time: 29.18m | eta: 33.0m +step 00710/01500 (47.33%) | loss: 2.447408 | lrm: 1.00 | dt: 2499.23ms | tok/sec: 209,780 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 16 | total time: 29.22m | eta: 33.0m +step 00711/01500 (47.40%) | loss: 2.485365 | lrm: 1.00 | dt: 2502.19ms | tok/sec: 209,531 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 16 | total time: 29.26m | eta: 32.9m +step 00712/01500 (47.47%) | loss: 2.525821 | lrm: 1.00 | dt: 2506.10ms | tok/sec: 209,204 | bf16_mfu: 31.94 | epoch: 1 pq: 8 rg: 24 | total time: 29.30m | eta: 32.9m +step 00713/01500 (47.53%) | loss: 2.563213 | lrm: 1.00 | dt: 2503.93ms | tok/sec: 209,385 | bf16_mfu: 31.97 | epoch: 1 pq: 8 rg: 24 | total time: 29.34m | eta: 32.9m +step 00714/01500 (47.60%) | loss: 2.450297 | lrm: 1.00 | dt: 2497.82ms | tok/sec: 209,898 | bf16_mfu: 32.05 | epoch: 1 pq: 8 rg: 24 | total time: 29.39m | eta: 32.8m +step 00715/01500 (47.67%) | loss: 2.448795 | lrm: 1.00 | dt: 2497.33ms | tok/sec: 209,939 | bf16_mfu: 32.05 | epoch: 1 pq: 8 rg: 24 | total time: 29.43m | eta: 32.8m +step 00716/01500 (47.73%) | loss: 2.484999 | lrm: 1.00 | dt: 2501.28ms | tok/sec: 209,607 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 24 | total time: 29.47m | eta: 32.7m +step 00717/01500 (47.80%) | loss: 2.458955 | lrm: 1.00 | dt: 2502.92ms | tok/sec: 209,470 | bf16_mfu: 31.98 | epoch: 1 pq: 8 rg: 24 | total time: 29.51m | eta: 32.7m +step 00718/01500 (47.87%) | loss: 2.459789 | lrm: 1.00 | dt: 2496.35ms | tok/sec: 210,021 | bf16_mfu: 32.06 | epoch: 1 pq: 8 rg: 24 | total time: 29.55m | eta: 32.6m +step 00719/01500 (47.93%) | loss: 2.461536 | lrm: 1.00 | dt: 2501.18ms | tok/sec: 209,615 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 24 | total time: 29.59m | eta: 32.6m +step 00720/01500 (48.00%) | loss: 2.508784 | lrm: 1.00 | dt: 2502.23ms | tok/sec: 209,527 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 24 | total time: 29.64m | eta: 32.6m +step 00721/01500 (48.07%) | loss: 2.546022 | lrm: 1.00 | dt: 2499.04ms | tok/sec: 209,795 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 32 | total time: 29.68m | eta: 32.5m +step 00722/01500 (48.13%) | loss: 2.569562 | lrm: 1.00 | dt: 2496.47ms | tok/sec: 210,011 | bf16_mfu: 32.06 | epoch: 1 pq: 8 rg: 32 | total time: 29.72m | eta: 32.5m +step 00723/01500 (48.20%) | loss: 2.549892 | lrm: 1.00 | dt: 2498.75ms | tok/sec: 209,820 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 32 | total time: 29.76m | eta: 32.4m +step 00724/01500 (48.27%) | loss: 2.553147 | lrm: 1.00 | dt: 2497.75ms | tok/sec: 209,904 | bf16_mfu: 32.05 | epoch: 1 pq: 8 rg: 32 | total time: 29.80m | eta: 32.4m +step 00725/01500 (48.33%) | loss: 2.513919 | lrm: 1.00 | dt: 2502.86ms | tok/sec: 209,475 | bf16_mfu: 31.98 | epoch: 1 pq: 8 rg: 32 | total time: 29.84m | eta: 32.3m +step 00726/01500 (48.40%) | loss: 2.511291 | lrm: 1.00 | dt: 2501.91ms | tok/sec: 209,555 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 32 | total time: 29.89m | eta: 32.3m +step 00727/01500 (48.47%) | loss: 2.526622 | lrm: 1.00 | dt: 2502.54ms | tok/sec: 209,502 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 32 | total time: 29.93m | eta: 32.3m +step 00728/01500 (48.53%) | loss: 2.583192 | lrm: 1.00 | dt: 2501.89ms | tok/sec: 209,556 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 40 | total time: 29.97m | eta: 32.2m +step 00729/01500 (48.60%) | loss: 2.570209 | lrm: 1.00 | dt: 2502.26ms | tok/sec: 209,525 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 40 | total time: 30.01m | eta: 32.2m +step 00730/01500 (48.67%) | loss: 2.555771 | lrm: 1.00 | dt: 2501.04ms | tok/sec: 209,627 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 40 | total time: 30.05m | eta: 32.1m +step 00731/01500 (48.73%) | loss: 2.594396 | lrm: 1.00 | dt: 2498.95ms | tok/sec: 209,803 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 40 | total time: 30.09m | eta: 32.1m +step 00732/01500 (48.80%) | loss: 2.598548 | lrm: 1.00 | dt: 2499.45ms | tok/sec: 209,761 | bf16_mfu: 32.03 | epoch: 1 pq: 8 rg: 40 | total time: 30.14m | eta: 32.1m +step 00733/01500 (48.87%) | loss: 2.575133 | lrm: 1.00 | dt: 2503.75ms | tok/sec: 209,401 | bf16_mfu: 31.97 | epoch: 1 pq: 8 rg: 40 | total time: 30.18m | eta: 32.0m +step 00734/01500 (48.93%) | loss: 2.570469 | lrm: 1.00 | dt: 2498.43ms | tok/sec: 209,847 | bf16_mfu: 32.04 | epoch: 1 pq: 8 rg: 40 | total time: 30.22m | eta: 32.0m +step 00735/01500 (49.00%) | loss: 2.577838 | lrm: 1.00 | dt: 2497.93ms | tok/sec: 209,889 | bf16_mfu: 32.04 | epoch: 1 pq: 8 rg: 40 | total time: 30.26m | eta: 31.9m +step 00736/01500 (49.07%) | loss: 2.562257 | lrm: 1.00 | dt: 2501.84ms | tok/sec: 209,560 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 48 | total time: 30.30m | eta: 31.9m +step 00737/01500 (49.13%) | loss: 2.512280 | lrm: 1.00 | dt: 2497.68ms | tok/sec: 209,910 | bf16_mfu: 32.05 | epoch: 1 pq: 8 rg: 48 | total time: 30.34m | eta: 31.8m +step 00738/01500 (49.20%) | loss: 2.525939 | lrm: 1.00 | dt: 2500.16ms | tok/sec: 209,701 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 48 | total time: 30.39m | eta: 31.8m +step 00739/01500 (49.27%) | loss: 2.487640 | lrm: 1.00 | dt: 2500.39ms | tok/sec: 209,682 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 48 | total time: 30.43m | eta: 31.8m +step 00740/01500 (49.33%) | loss: 2.484023 | lrm: 1.00 | dt: 2503.10ms | tok/sec: 209,455 | bf16_mfu: 31.98 | epoch: 1 pq: 8 rg: 48 | total time: 30.47m | eta: 31.7m +step 00741/01500 (49.40%) | loss: 2.495674 | lrm: 1.00 | dt: 2505.57ms | tok/sec: 209,249 | bf16_mfu: 31.95 | epoch: 1 pq: 8 rg: 48 | total time: 30.51m | eta: 31.7m +step 00742/01500 (49.47%) | loss: 2.493286 | lrm: 1.00 | dt: 2500.44ms | tok/sec: 209,678 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 48 | total time: 30.55m | eta: 31.6m +step 00743/01500 (49.53%) | loss: 2.512838 | lrm: 1.00 | dt: 2506.23ms | tok/sec: 209,193 | bf16_mfu: 31.94 | epoch: 1 pq: 8 rg: 48 | total time: 30.59m | eta: 31.6m +step 00744/01500 (49.60%) | loss: 2.526144 | lrm: 1.00 | dt: 2499.91ms | tok/sec: 209,722 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 48 | total time: 30.64m | eta: 31.6m +step 00745/01500 (49.67%) | loss: 2.485605 | lrm: 1.00 | dt: 2502.44ms | tok/sec: 209,511 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 56 | total time: 30.68m | eta: 31.5m +step 00746/01500 (49.73%) | loss: 2.502808 | lrm: 1.00 | dt: 2496.03ms | tok/sec: 210,048 | bf16_mfu: 32.07 | epoch: 1 pq: 8 rg: 56 | total time: 30.72m | eta: 31.5m +step 00747/01500 (49.80%) | loss: 2.541489 | lrm: 1.00 | dt: 2500.48ms | tok/sec: 209,674 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 56 | total time: 30.76m | eta: 31.4m +step 00748/01500 (49.87%) | loss: 2.580649 | lrm: 1.00 | dt: 2500.65ms | tok/sec: 209,660 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 56 | total time: 30.80m | eta: 31.4m +step 00749/01500 (49.93%) | loss: 2.583109 | lrm: 1.00 | dt: 2500.45ms | tok/sec: 209,677 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 56 | total time: 30.84m | eta: 31.3m +<|bos|>The capital of France is Paris. The city is the capital of France and is the capital of the Î +<|bos|>The chemical symbol of gold is Au. It is a chemical element with the symbol Au. It is a chemical +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky and the ocean. I love the ocean because it +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +2026-08-10 22:10:01,490 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank1.pt +2026-08-10 22:10:14,777 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank6.pt +2026-08-10 22:10:18,203 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank3.pt +2026-08-10 22:10:19,319 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank7.pt +2026-08-10 22:10:20,280 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank2.pt +2026-08-10 22:10:23,249 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank4.pt +2026-08-10 22:10:24,164 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank5.pt +2026-08-10 22:11:24,553 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_000750.pt +2026-08-10 22:11:24,556 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_000750.json +2026-08-10 22:11:31,788 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_000750_rank0.pt +step 00750/01500 (50.00%) | loss: 2.613416 | lrm: 1.00 | dt: 2475.46ms | tok/sec: 211,794 | bf16_mfu: 32.34 | epoch: 1 pq: 8 rg: 56 | total time: 30.89m | eta: 31.3m +step 00751/01500 (50.07%) | loss: 2.575135 | lrm: 1.00 | dt: 2483.76ms | tok/sec: 211,086 | bf16_mfu: 32.23 | epoch: 1 pq: 8 rg: 56 | total time: 30.93m | eta: 31.3m +step 00752/01500 (50.13%) | loss: 2.561207 | lrm: 1.00 | dt: 2488.51ms | tok/sec: 210,683 | bf16_mfu: 32.17 | epoch: 1 pq: 8 rg: 56 | total time: 30.97m | eta: 31.2m +step 00753/01500 (50.20%) | loss: 2.551033 | lrm: 1.00 | dt: 2489.22ms | tok/sec: 210,623 | bf16_mfu: 32.16 | epoch: 1 pq: 8 rg: 64 | total time: 31.01m | eta: 31.2m +step 00754/01500 (50.27%) | loss: 2.522961 | lrm: 1.00 | dt: 2494.46ms | tok/sec: 210,181 | bf16_mfu: 32.09 | epoch: 1 pq: 8 rg: 64 | total time: 31.05m | eta: 31.1m +step 00755/01500 (50.33%) | loss: 2.514381 | lrm: 1.00 | dt: 2496.97ms | tok/sec: 209,969 | bf16_mfu: 32.06 | epoch: 1 pq: 8 rg: 64 | total time: 31.09m | eta: 31.1m +step 00756/01500 (50.40%) | loss: 2.539011 | lrm: 1.00 | dt: 2498.62ms | tok/sec: 209,831 | bf16_mfu: 32.04 | epoch: 1 pq: 8 rg: 64 | total time: 31.13m | eta: 31.1m +step 00757/01500 (50.47%) | loss: 2.512881 | lrm: 1.00 | dt: 2496.05ms | tok/sec: 210,046 | bf16_mfu: 32.07 | epoch: 1 pq: 8 rg: 64 | total time: 31.18m | eta: 31.0m +step 00758/01500 (50.53%) | loss: 2.500824 | lrm: 1.00 | dt: 2506.59ms | tok/sec: 209,164 | bf16_mfu: 31.93 | epoch: 1 pq: 8 rg: 64 | total time: 31.22m | eta: 31.0m +step 00759/01500 (50.60%) | loss: 2.456874 | lrm: 1.00 | dt: 2510.10ms | tok/sec: 208,870 | bf16_mfu: 31.89 | epoch: 1 pq: 8 rg: 64 | total time: 31.26m | eta: 30.9m +step 00760/01500 (50.67%) | loss: 2.449276 | lrm: 1.00 | dt: 2505.61ms | tok/sec: 209,245 | bf16_mfu: 31.95 | epoch: 1 pq: 8 rg: 64 | total time: 31.30m | eta: 30.9m +step 00761/01500 (50.73%) | loss: 2.460873 | lrm: 1.00 | dt: 2511.35ms | tok/sec: 208,767 | bf16_mfu: 31.87 | epoch: 1 pq: 8 rg: 72 | total time: 31.34m | eta: 30.8m +step 00762/01500 (50.80%) | loss: 2.463210 | lrm: 1.00 | dt: 2503.80ms | tok/sec: 209,397 | bf16_mfu: 31.97 | epoch: 1 pq: 8 rg: 72 | total time: 31.39m | eta: 30.8m +step 00763/01500 (50.87%) | loss: 2.438271 | lrm: 1.00 | dt: 2504.37ms | tok/sec: 209,349 | bf16_mfu: 31.96 | epoch: 1 pq: 8 rg: 72 | total time: 31.43m | eta: 30.8m +step 00764/01500 (50.93%) | loss: 2.398702 | lrm: 1.00 | dt: 2511.76ms | tok/sec: 208,733 | bf16_mfu: 31.87 | epoch: 1 pq: 8 rg: 72 | total time: 31.47m | eta: 30.7m +step 00765/01500 (51.00%) | loss: 2.378540 | lrm: 1.00 | dt: 2504.58ms | tok/sec: 209,331 | bf16_mfu: 31.96 | epoch: 1 pq: 8 rg: 72 | total time: 31.51m | eta: 30.7m +step 00766/01500 (51.07%) | loss: 2.396469 | lrm: 1.00 | dt: 2506.18ms | tok/sec: 209,198 | bf16_mfu: 31.94 | epoch: 1 pq: 8 rg: 72 | total time: 31.55m | eta: 30.6m +step 00767/01500 (51.13%) | loss: 2.431147 | lrm: 1.00 | dt: 2501.81ms | tok/sec: 209,563 | bf16_mfu: 32.00 | epoch: 1 pq: 8 rg: 72 | total time: 31.59m | eta: 30.6m +step 00768/01500 (51.20%) | loss: 2.426193 | lrm: 1.00 | dt: 2500.47ms | tok/sec: 209,675 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 72 | total time: 31.64m | eta: 30.6m +step 00769/01500 (51.27%) | loss: 2.418622 | lrm: 1.00 | dt: 2503.90ms | tok/sec: 209,388 | bf16_mfu: 31.97 | epoch: 1 pq: 8 rg: 80 | total time: 31.68m | eta: 30.5m +step 00770/01500 (51.33%) | loss: 2.455873 | lrm: 1.00 | dt: 2497.98ms | tok/sec: 209,884 | bf16_mfu: 32.04 | epoch: 1 pq: 8 rg: 80 | total time: 31.72m | eta: 30.5m +step 00771/01500 (51.40%) | loss: 2.489749 | lrm: 1.00 | dt: 2500.25ms | tok/sec: 209,694 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 80 | total time: 31.76m | eta: 30.4m +step 00772/01500 (51.47%) | loss: 2.477514 | lrm: 1.00 | dt: 2502.03ms | tok/sec: 209,544 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 80 | total time: 31.80m | eta: 30.4m +step 00773/01500 (51.53%) | loss: 2.490943 | lrm: 1.00 | dt: 2499.93ms | tok/sec: 209,721 | bf16_mfu: 32.02 | epoch: 1 pq: 8 rg: 80 | total time: 31.84m | eta: 30.3m +step 00774/01500 (51.60%) | loss: 2.454402 | lrm: 1.00 | dt: 2500.35ms | tok/sec: 209,685 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 80 | total time: 31.89m | eta: 30.3m +step 00775/01500 (51.67%) | loss: 2.453161 | lrm: 1.00 | dt: 2500.67ms | tok/sec: 209,659 | bf16_mfu: 32.01 | epoch: 1 pq: 8 rg: 80 | total time: 31.93m | eta: 30.3m +step 00776/01500 (51.73%) | loss: 2.457556 | lrm: 1.00 | dt: 2502.05ms | tok/sec: 209,543 | bf16_mfu: 31.99 | epoch: 1 pq: 8 rg: 80 | total time: 31.97m | eta: 30.2m +step 00777/01500 (51.80%) | loss: 2.470391 | lrm: 1.00 | dt: 2496.88ms | tok/sec: 209,976 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 0 | total time: 32.01m | eta: 30.2m +step 00778/01500 (51.87%) | loss: 2.436660 | lrm: 1.00 | dt: 2500.80ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 0 | total time: 32.05m | eta: 30.1m +step 00779/01500 (51.93%) | loss: 2.393297 | lrm: 1.00 | dt: 2497.66ms | tok/sec: 209,912 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 0 | total time: 32.09m | eta: 30.1m +step 00780/01500 (52.00%) | loss: 2.393302 | lrm: 1.00 | dt: 2496.59ms | tok/sec: 210,001 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 0 | total time: 32.14m | eta: 30.0m +step 00781/01500 (52.07%) | loss: 2.438059 | lrm: 1.00 | dt: 2499.50ms | tok/sec: 209,757 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 0 | total time: 32.18m | eta: 30.0m +step 00782/01500 (52.13%) | loss: 2.459424 | lrm: 1.00 | dt: 2503.23ms | tok/sec: 209,444 | bf16_mfu: 31.98 | epoch: 1 pq: 9 rg: 0 | total time: 32.22m | eta: 30.0m +step 00783/01500 (52.20%) | loss: 2.507362 | lrm: 1.00 | dt: 2499.13ms | tok/sec: 209,788 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 0 | total time: 32.26m | eta: 29.9m +step 00784/01500 (52.27%) | loss: 2.479795 | lrm: 1.00 | dt: 2796.38ms | tok/sec: 187,487 | bf16_mfu: 28.62 | epoch: 1 pq: 9 rg: 0 | total time: 32.31m | eta: 29.9m +step 00785/01500 (52.33%) | loss: 2.443444 | lrm: 1.00 | dt: 2500.97ms | tok/sec: 209,634 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 8 | total time: 32.35m | eta: 29.8m +step 00786/01500 (52.40%) | loss: 2.376552 | lrm: 1.00 | dt: 2499.26ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 8 | total time: 32.39m | eta: 29.8m +step 00787/01500 (52.47%) | loss: 2.346382 | lrm: 1.00 | dt: 2496.73ms | tok/sec: 209,989 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 8 | total time: 32.43m | eta: 29.8m +step 00788/01500 (52.53%) | loss: 2.339653 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 8 | total time: 32.47m | eta: 29.7m +step 00789/01500 (52.60%) | loss: 2.385250 | lrm: 1.00 | dt: 2503.26ms | tok/sec: 209,441 | bf16_mfu: 31.98 | epoch: 1 pq: 9 rg: 8 | total time: 32.52m | eta: 29.7m +step 00790/01500 (52.67%) | loss: 2.442078 | lrm: 1.00 | dt: 2499.15ms | tok/sec: 209,786 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 8 | total time: 32.56m | eta: 29.6m +step 00791/01500 (52.73%) | loss: 2.409874 | lrm: 1.00 | dt: 2499.38ms | tok/sec: 209,767 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 8 | total time: 32.60m | eta: 29.6m +step 00792/01500 (52.80%) | loss: 2.374510 | lrm: 1.00 | dt: 2498.75ms | tok/sec: 209,820 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 8 | total time: 32.64m | eta: 29.6m +step 00793/01500 (52.87%) | loss: 2.398126 | lrm: 1.00 | dt: 2498.92ms | tok/sec: 209,805 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 16 | total time: 32.68m | eta: 29.5m +step 00794/01500 (52.93%) | loss: 2.397676 | lrm: 1.00 | dt: 2499.77ms | tok/sec: 209,734 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 16 | total time: 32.72m | eta: 29.5m +step 00795/01500 (53.00%) | loss: 2.424432 | lrm: 1.00 | dt: 2500.29ms | tok/sec: 209,690 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 16 | total time: 32.77m | eta: 29.4m +step 00796/01500 (53.07%) | loss: 2.421531 | lrm: 1.00 | dt: 2500.98ms | tok/sec: 209,633 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 16 | total time: 32.81m | eta: 29.4m +step 00797/01500 (53.13%) | loss: 2.430554 | lrm: 1.00 | dt: 2500.53ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 16 | total time: 32.85m | eta: 29.3m +step 00798/01500 (53.20%) | loss: 2.479384 | lrm: 1.00 | dt: 2496.81ms | tok/sec: 209,982 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 16 | total time: 32.89m | eta: 29.3m +step 00799/01500 (53.27%) | loss: 2.448715 | lrm: 1.00 | dt: 2501.10ms | tok/sec: 209,623 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 16 | total time: 32.93m | eta: 29.3m +Step 00800 | Validation bpb: 0.741616 +Step 00800 | ICL early: 2.7298 late: 2.8802 score: +0.1504 | Induction loss: 9.6485 acc: 0.0788 (random-half: 13.9446) +step 00800/01500 (53.33%) | loss: 2.464734 | lrm: 1.00 | dt: 2497.15ms | tok/sec: 209,954 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 16 | total time: 32.97m | eta: 29.2m +step 00801/01500 (53.40%) | loss: 2.485478 | lrm: 1.00 | dt: 2497.07ms | tok/sec: 209,961 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 24 | total time: 33.02m | eta: 29.2m +step 00802/01500 (53.47%) | loss: 2.503597 | lrm: 1.00 | dt: 2512.89ms | tok/sec: 208,639 | bf16_mfu: 31.85 | epoch: 1 pq: 9 rg: 24 | total time: 33.06m | eta: 29.1m +step 00803/01500 (53.53%) | loss: 2.497169 | lrm: 1.00 | dt: 2495.10ms | tok/sec: 210,126 | bf16_mfu: 32.08 | epoch: 1 pq: 9 rg: 24 | total time: 33.10m | eta: 29.1m +step 00804/01500 (53.60%) | loss: 2.495256 | lrm: 1.00 | dt: 2498.36ms | tok/sec: 209,853 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 24 | total time: 33.14m | eta: 29.1m +step 00805/01500 (53.67%) | loss: 2.459168 | lrm: 1.00 | dt: 2496.95ms | tok/sec: 209,971 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 24 | total time: 33.18m | eta: 29.0m +step 00806/01500 (53.73%) | loss: 2.542786 | lrm: 1.00 | dt: 2499.63ms | tok/sec: 209,746 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 24 | total time: 33.22m | eta: 29.0m +step 00807/01500 (53.80%) | loss: 2.502023 | lrm: 1.00 | dt: 2498.68ms | tok/sec: 209,826 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 24 | total time: 33.27m | eta: 28.9m +step 00808/01500 (53.87%) | loss: 2.530503 | lrm: 1.00 | dt: 2498.87ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 24 | total time: 33.31m | eta: 28.9m +step 00809/01500 (53.93%) | loss: 2.506526 | lrm: 1.00 | dt: 2499.72ms | tok/sec: 209,738 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 24 | total time: 33.35m | eta: 28.8m +step 00810/01500 (54.00%) | loss: 2.499926 | lrm: 1.00 | dt: 2499.17ms | tok/sec: 209,784 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 32 | total time: 33.39m | eta: 28.8m +step 00811/01500 (54.07%) | loss: 2.511620 | lrm: 1.00 | dt: 2499.70ms | tok/sec: 209,740 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 32 | total time: 33.43m | eta: 28.8m +step 00812/01500 (54.13%) | loss: 2.518845 | lrm: 1.00 | dt: 2500.26ms | tok/sec: 209,693 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 32 | total time: 33.47m | eta: 28.7m +step 00813/01500 (54.20%) | loss: 2.486244 | lrm: 1.00 | dt: 2497.50ms | tok/sec: 209,925 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 32 | total time: 33.52m | eta: 28.7m +step 00814/01500 (54.27%) | loss: 2.483264 | lrm: 1.00 | dt: 2502.02ms | tok/sec: 209,545 | bf16_mfu: 31.99 | epoch: 1 pq: 9 rg: 32 | total time: 33.56m | eta: 28.6m +step 00815/01500 (54.33%) | loss: 2.496015 | lrm: 1.00 | dt: 2500.60ms | tok/sec: 209,665 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 32 | total time: 33.60m | eta: 28.6m +step 00816/01500 (54.40%) | loss: 2.470131 | lrm: 1.00 | dt: 2501.89ms | tok/sec: 209,556 | bf16_mfu: 31.99 | epoch: 1 pq: 9 rg: 32 | total time: 33.64m | eta: 28.5m +step 00817/01500 (54.47%) | loss: 2.521402 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 32 | total time: 33.68m | eta: 28.5m +step 00818/01500 (54.53%) | loss: 2.530285 | lrm: 1.00 | dt: 2497.64ms | tok/sec: 209,913 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 40 | total time: 33.72m | eta: 28.5m +step 00819/01500 (54.60%) | loss: 2.537301 | lrm: 1.00 | dt: 2504.44ms | tok/sec: 209,343 | bf16_mfu: 31.96 | epoch: 1 pq: 9 rg: 40 | total time: 33.77m | eta: 28.4m +step 00820/01500 (54.67%) | loss: 2.521476 | lrm: 1.00 | dt: 2498.51ms | tok/sec: 209,840 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 40 | total time: 33.81m | eta: 28.4m +step 00821/01500 (54.73%) | loss: 2.536437 | lrm: 1.00 | dt: 2500.13ms | tok/sec: 209,704 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 40 | total time: 33.85m | eta: 28.3m +step 00822/01500 (54.80%) | loss: 2.523154 | lrm: 1.00 | dt: 2501.26ms | tok/sec: 209,609 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 40 | total time: 33.89m | eta: 28.3m +step 00823/01500 (54.87%) | loss: 2.503628 | lrm: 1.00 | dt: 2505.43ms | tok/sec: 209,260 | bf16_mfu: 31.95 | epoch: 1 pq: 9 rg: 40 | total time: 33.93m | eta: 28.3m +step 00824/01500 (54.93%) | loss: 2.489208 | lrm: 1.00 | dt: 2497.59ms | tok/sec: 209,917 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 40 | total time: 33.97m | eta: 28.2m +step 00825/01500 (55.00%) | loss: 2.499214 | lrm: 1.00 | dt: 2499.67ms | tok/sec: 209,742 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 40 | total time: 34.02m | eta: 28.2m +step 00826/01500 (55.07%) | loss: 2.467831 | lrm: 1.00 | dt: 2499.25ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 48 | total time: 34.06m | eta: 28.1m +step 00827/01500 (55.13%) | loss: 2.434501 | lrm: 1.00 | dt: 2498.28ms | tok/sec: 209,859 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 48 | total time: 34.10m | eta: 28.1m +step 00828/01500 (55.20%) | loss: 2.413911 | lrm: 1.00 | dt: 2500.52ms | tok/sec: 209,671 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 48 | total time: 34.14m | eta: 28.0m +step 00829/01500 (55.27%) | loss: 2.417626 | lrm: 1.00 | dt: 2497.00ms | tok/sec: 209,966 | bf16_mfu: 32.06 | epoch: 1 pq: 9 rg: 48 | total time: 34.18m | eta: 28.0m +step 00830/01500 (55.33%) | loss: 2.412850 | lrm: 1.00 | dt: 2500.08ms | tok/sec: 209,708 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 48 | total time: 34.22m | eta: 28.0m +step 00831/01500 (55.40%) | loss: 2.401505 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 48 | total time: 34.27m | eta: 27.9m +step 00832/01500 (55.47%) | loss: 2.387107 | lrm: 1.00 | dt: 2500.33ms | tok/sec: 209,687 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 48 | total time: 34.31m | eta: 27.9m +step 00833/01500 (55.53%) | loss: 2.388825 | lrm: 1.00 | dt: 2503.41ms | tok/sec: 209,429 | bf16_mfu: 31.97 | epoch: 1 pq: 9 rg: 48 | total time: 34.35m | eta: 27.8m +step 00834/01500 (55.60%) | loss: 2.321134 | lrm: 1.00 | dt: 2502.12ms | tok/sec: 209,537 | bf16_mfu: 31.99 | epoch: 1 pq: 9 rg: 56 | total time: 34.39m | eta: 27.8m +step 00835/01500 (55.67%) | loss: 2.310482 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,811 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 56 | total time: 34.43m | eta: 27.8m +step 00836/01500 (55.73%) | loss: 2.296415 | lrm: 1.00 | dt: 2498.59ms | tok/sec: 209,833 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 56 | total time: 34.47m | eta: 27.7m +step 00837/01500 (55.80%) | loss: 2.241724 | lrm: 1.00 | dt: 2497.20ms | tok/sec: 209,950 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 56 | total time: 34.52m | eta: 27.7m +step 00838/01500 (55.87%) | loss: 2.274436 | lrm: 1.00 | dt: 2497.66ms | tok/sec: 209,911 | bf16_mfu: 32.05 | epoch: 1 pq: 9 rg: 56 | total time: 34.56m | eta: 27.6m +step 00839/01500 (55.93%) | loss: 2.280051 | lrm: 1.00 | dt: 2498.69ms | tok/sec: 209,825 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 56 | total time: 34.60m | eta: 27.6m +step 00840/01500 (56.00%) | loss: 2.259181 | lrm: 1.00 | dt: 2501.94ms | tok/sec: 209,552 | bf16_mfu: 31.99 | epoch: 1 pq: 9 rg: 56 | total time: 34.64m | eta: 27.5m +step 00841/01500 (56.07%) | loss: 2.319050 | lrm: 1.00 | dt: 2498.99ms | tok/sec: 209,799 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 56 | total time: 34.68m | eta: 27.5m +step 00842/01500 (56.13%) | loss: 2.335775 | lrm: 1.00 | dt: 2498.74ms | tok/sec: 209,820 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 64 | total time: 34.72m | eta: 27.5m +step 00843/01500 (56.20%) | loss: 2.365811 | lrm: 1.00 | dt: 2500.41ms | tok/sec: 209,680 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 64 | total time: 34.77m | eta: 27.4m +step 00844/01500 (56.27%) | loss: 2.361053 | lrm: 1.00 | dt: 2501.03ms | tok/sec: 209,628 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 64 | total time: 34.81m | eta: 27.4m +step 00845/01500 (56.33%) | loss: 2.376585 | lrm: 1.00 | dt: 2505.31ms | tok/sec: 209,270 | bf16_mfu: 31.95 | epoch: 1 pq: 9 rg: 64 | total time: 34.85m | eta: 27.3m +step 00846/01500 (56.40%) | loss: 2.384495 | lrm: 1.00 | dt: 2502.76ms | tok/sec: 209,483 | bf16_mfu: 31.98 | epoch: 1 pq: 9 rg: 64 | total time: 34.89m | eta: 27.3m +step 00847/01500 (56.47%) | loss: 2.378119 | lrm: 1.00 | dt: 2505.06ms | tok/sec: 209,291 | bf16_mfu: 31.95 | epoch: 1 pq: 9 rg: 64 | total time: 34.93m | eta: 27.3m +step 00848/01500 (56.53%) | loss: 2.429506 | lrm: 1.00 | dt: 2501.69ms | tok/sec: 209,573 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 64 | total time: 34.97m | eta: 27.2m +step 00849/01500 (56.60%) | loss: 2.447713 | lrm: 1.00 | dt: 2503.76ms | tok/sec: 209,400 | bf16_mfu: 31.97 | epoch: 1 pq: 9 rg: 64 | total time: 35.02m | eta: 27.2m +step 00850/01500 (56.67%) | loss: 2.429006 | lrm: 1.00 | dt: 2502.71ms | tok/sec: 209,488 | bf16_mfu: 31.98 | epoch: 1 pq: 9 rg: 72 | total time: 35.06m | eta: 27.1m +step 00851/01500 (56.73%) | loss: 2.430386 | lrm: 1.00 | dt: 2501.07ms | tok/sec: 209,625 | bf16_mfu: 32.00 | epoch: 1 pq: 9 rg: 72 | total time: 35.10m | eta: 27.1m +step 00852/01500 (56.80%) | loss: 2.461675 | lrm: 1.00 | dt: 2500.50ms | tok/sec: 209,672 | bf16_mfu: 32.01 | epoch: 1 pq: 9 rg: 72 | total time: 35.14m | eta: 27.0m +step 00853/01500 (56.87%) | loss: 2.427366 | lrm: 1.00 | dt: 2497.99ms | tok/sec: 209,883 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 72 | total time: 35.18m | eta: 27.0m +step 00854/01500 (56.93%) | loss: 2.420453 | lrm: 1.00 | dt: 2500.12ms | tok/sec: 209,705 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 72 | total time: 35.22m | eta: 27.0m +step 00855/01500 (57.00%) | loss: 2.422431 | lrm: 1.00 | dt: 2506.84ms | tok/sec: 209,142 | bf16_mfu: 31.93 | epoch: 1 pq: 9 rg: 72 | total time: 35.27m | eta: 26.9m +step 00856/01500 (57.07%) | loss: 2.393103 | lrm: 1.00 | dt: 2504.09ms | tok/sec: 209,372 | bf16_mfu: 31.97 | epoch: 1 pq: 9 rg: 72 | total time: 35.31m | eta: 26.9m +step 00857/01500 (57.13%) | loss: 2.413194 | lrm: 1.00 | dt: 2499.06ms | tok/sec: 209,794 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 72 | total time: 35.35m | eta: 26.8m +step 00858/01500 (57.20%) | loss: 2.432611 | lrm: 1.00 | dt: 2498.59ms | tok/sec: 209,833 | bf16_mfu: 32.04 | epoch: 1 pq: 9 rg: 80 | total time: 35.39m | eta: 26.8m +step 00859/01500 (57.27%) | loss: 2.446908 | lrm: 1.00 | dt: 2505.13ms | tok/sec: 209,286 | bf16_mfu: 31.95 | epoch: 1 pq: 9 rg: 80 | total time: 35.43m | eta: 26.8m +step 00860/01500 (57.33%) | loss: 2.430805 | lrm: 1.00 | dt: 2503.85ms | tok/sec: 209,392 | bf16_mfu: 31.97 | epoch: 1 pq: 9 rg: 80 | total time: 35.47m | eta: 26.7m +step 00861/01500 (57.40%) | loss: 2.454606 | lrm: 1.00 | dt: 2499.66ms | tok/sec: 209,743 | bf16_mfu: 32.02 | epoch: 1 pq: 9 rg: 80 | total time: 35.52m | eta: 26.7m +step 00862/01500 (57.47%) | loss: 2.432909 | lrm: 1.00 | dt: 2504.19ms | tok/sec: 209,364 | bf16_mfu: 31.96 | epoch: 1 pq: 9 rg: 80 | total time: 35.56m | eta: 26.6m +step 00863/01500 (57.53%) | loss: 2.460640 | lrm: 1.00 | dt: 2498.95ms | tok/sec: 209,803 | bf16_mfu: 32.03 | epoch: 1 pq: 9 rg: 80 | total time: 35.60m | eta: 26.6m +step 00864/01500 (57.60%) | loss: 2.466855 | lrm: 1.00 | dt: 2495.95ms | tok/sec: 210,055 | bf16_mfu: 32.07 | epoch: 1 pq: 9 rg: 80 | total time: 35.64m | eta: 26.5m +step 00865/01500 (57.67%) | loss: 2.488819 | lrm: 1.00 | dt: 2751.02ms | tok/sec: 190,579 | bf16_mfu: 29.10 | epoch: 1 pq: 9 rg: 80 | total time: 35.69m | eta: 26.5m +step 00866/01500 (57.73%) | loss: 2.460920 | lrm: 1.00 | dt: 2502.21ms | tok/sec: 209,529 | bf16_mfu: 31.99 | epoch: 1 pq: 10 rg: 0 | total time: 35.73m | eta: 26.5m +step 00867/01500 (57.80%) | loss: 2.434904 | lrm: 1.00 | dt: 2503.90ms | tok/sec: 209,388 | bf16_mfu: 31.97 | epoch: 1 pq: 10 rg: 0 | total time: 35.77m | eta: 26.4m +step 00868/01500 (57.87%) | loss: 2.458098 | lrm: 1.00 | dt: 2513.55ms | tok/sec: 208,584 | bf16_mfu: 31.85 | epoch: 1 pq: 10 rg: 0 | total time: 35.81m | eta: 26.4m +step 00869/01500 (57.93%) | loss: 2.449237 | lrm: 1.00 | dt: 2497.80ms | tok/sec: 209,900 | bf16_mfu: 32.05 | epoch: 1 pq: 10 rg: 0 | total time: 35.85m | eta: 26.3m +step 00870/01500 (58.00%) | loss: 2.448500 | lrm: 1.00 | dt: 2497.95ms | tok/sec: 209,887 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 0 | total time: 35.90m | eta: 26.3m +step 00871/01500 (58.07%) | loss: 2.446891 | lrm: 1.00 | dt: 2500.76ms | tok/sec: 209,651 | bf16_mfu: 32.01 | epoch: 1 pq: 10 rg: 0 | total time: 35.94m | eta: 26.3m +step 00872/01500 (58.13%) | loss: 2.394001 | lrm: 1.00 | dt: 2496.83ms | tok/sec: 209,981 | bf16_mfu: 32.06 | epoch: 1 pq: 10 rg: 0 | total time: 35.98m | eta: 26.2m +step 00873/01500 (58.20%) | loss: 2.374766 | lrm: 1.00 | dt: 2502.69ms | tok/sec: 209,489 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 0 | total time: 36.02m | eta: 26.2m +step 00874/01500 (58.27%) | loss: 2.393871 | lrm: 1.00 | dt: 2499.73ms | tok/sec: 209,737 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 8 | total time: 36.06m | eta: 26.1m +step 00875/01500 (58.33%) | loss: 2.388008 | lrm: 1.00 | dt: 2502.98ms | tok/sec: 209,465 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 8 | total time: 36.10m | eta: 26.1m +step 00876/01500 (58.40%) | loss: 2.403612 | lrm: 1.00 | dt: 2499.08ms | tok/sec: 209,792 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 8 | total time: 36.15m | eta: 26.0m +step 00877/01500 (58.47%) | loss: 2.423884 | lrm: 1.00 | dt: 2497.25ms | tok/sec: 209,945 | bf16_mfu: 32.05 | epoch: 1 pq: 10 rg: 8 | total time: 36.19m | eta: 26.0m +step 00878/01500 (58.53%) | loss: 2.443265 | lrm: 1.00 | dt: 2504.74ms | tok/sec: 209,318 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 8 | total time: 36.23m | eta: 26.0m +step 00879/01500 (58.60%) | loss: 2.426598 | lrm: 1.00 | dt: 2495.43ms | tok/sec: 210,099 | bf16_mfu: 32.08 | epoch: 1 pq: 10 rg: 8 | total time: 36.27m | eta: 25.9m +step 00880/01500 (58.67%) | loss: 2.472516 | lrm: 1.00 | dt: 2498.87ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 8 | total time: 36.31m | eta: 25.9m +step 00881/01500 (58.73%) | loss: 2.499515 | lrm: 1.00 | dt: 2500.67ms | tok/sec: 209,658 | bf16_mfu: 32.01 | epoch: 1 pq: 10 rg: 8 | total time: 36.35m | eta: 25.8m +step 00882/01500 (58.80%) | loss: 2.513423 | lrm: 1.00 | dt: 2499.99ms | tok/sec: 209,715 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 8 | total time: 36.40m | eta: 25.8m +step 00883/01500 (58.87%) | loss: 2.552547 | lrm: 1.00 | dt: 2502.99ms | tok/sec: 209,465 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 16 | total time: 36.44m | eta: 25.8m +step 00884/01500 (58.93%) | loss: 2.552526 | lrm: 1.00 | dt: 2504.21ms | tok/sec: 209,362 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 16 | total time: 36.48m | eta: 25.7m +step 00885/01500 (59.00%) | loss: 2.534397 | lrm: 1.00 | dt: 2501.25ms | tok/sec: 209,610 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 16 | total time: 36.52m | eta: 25.7m +step 00886/01500 (59.07%) | loss: 2.541211 | lrm: 1.00 | dt: 2500.06ms | tok/sec: 209,710 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 16 | total time: 36.56m | eta: 25.6m +step 00887/01500 (59.13%) | loss: 2.519362 | lrm: 1.00 | dt: 2499.03ms | tok/sec: 209,796 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 16 | total time: 36.60m | eta: 25.6m +step 00888/01500 (59.20%) | loss: 2.499014 | lrm: 1.00 | dt: 2501.62ms | tok/sec: 209,579 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 16 | total time: 36.65m | eta: 25.5m +step 00889/01500 (59.27%) | loss: 2.477072 | lrm: 1.00 | dt: 2503.42ms | tok/sec: 209,428 | bf16_mfu: 31.97 | epoch: 1 pq: 10 rg: 16 | total time: 36.69m | eta: 25.5m +step 00890/01500 (59.33%) | loss: 2.486483 | lrm: 1.00 | dt: 2500.29ms | tok/sec: 209,690 | bf16_mfu: 32.01 | epoch: 1 pq: 10 rg: 16 | total time: 36.73m | eta: 25.5m +step 00891/01500 (59.40%) | loss: 2.499196 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 24 | total time: 36.77m | eta: 25.4m +step 00892/01500 (59.47%) | loss: 2.511819 | lrm: 1.00 | dt: 2497.76ms | tok/sec: 209,903 | bf16_mfu: 32.05 | epoch: 1 pq: 10 rg: 24 | total time: 36.81m | eta: 25.4m +step 00893/01500 (59.53%) | loss: 2.506047 | lrm: 1.00 | dt: 2498.67ms | tok/sec: 209,826 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 24 | total time: 36.85m | eta: 25.3m +step 00894/01500 (59.60%) | loss: 2.506997 | lrm: 1.00 | dt: 2504.83ms | tok/sec: 209,310 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 24 | total time: 36.90m | eta: 25.3m +step 00895/01500 (59.67%) | loss: 2.437824 | lrm: 1.00 | dt: 2494.00ms | tok/sec: 210,219 | bf16_mfu: 32.10 | epoch: 1 pq: 10 rg: 24 | total time: 36.94m | eta: 25.3m +step 00896/01500 (59.73%) | loss: 2.491216 | lrm: 1.00 | dt: 2500.11ms | tok/sec: 209,706 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 24 | total time: 36.98m | eta: 25.2m +step 00897/01500 (59.80%) | loss: 2.459149 | lrm: 1.00 | dt: 2506.67ms | tok/sec: 209,157 | bf16_mfu: 31.93 | epoch: 1 pq: 10 rg: 24 | total time: 37.02m | eta: 25.2m +step 00898/01500 (59.87%) | loss: 2.434871 | lrm: 1.00 | dt: 2497.20ms | tok/sec: 209,950 | bf16_mfu: 32.05 | epoch: 1 pq: 10 rg: 24 | total time: 37.06m | eta: 25.1m +step 00899/01500 (59.93%) | loss: 2.421553 | lrm: 1.00 | dt: 2498.61ms | tok/sec: 209,831 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 32 | total time: 37.10m | eta: 25.1m +Step 00900 | Validation bpb: 0.738064 +Step 00900 | ICL early: 2.7088 late: 2.8737 score: +0.1649 | Induction loss: 9.3294 acc: 0.0943 (random-half: 13.9594) +step 00900/01500 (60.00%) | loss: 2.437313 | lrm: 1.00 | dt: 2495.98ms | tok/sec: 210,053 | bf16_mfu: 32.07 | epoch: 1 pq: 10 rg: 32 | total time: 37.15m | eta: 25.0m +step 00901/01500 (60.07%) | loss: 2.452271 | lrm: 1.00 | dt: 2499.60ms | tok/sec: 209,748 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 32 | total time: 37.19m | eta: 25.0m +step 00902/01500 (60.13%) | loss: 2.442853 | lrm: 1.00 | dt: 2496.67ms | tok/sec: 209,995 | bf16_mfu: 32.06 | epoch: 1 pq: 10 rg: 32 | total time: 37.23m | eta: 25.0m +step 00903/01500 (60.20%) | loss: 2.442931 | lrm: 1.00 | dt: 2499.68ms | tok/sec: 209,742 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 32 | total time: 37.27m | eta: 24.9m +step 00904/01500 (60.27%) | loss: 2.466908 | lrm: 1.00 | dt: 2496.53ms | tok/sec: 210,006 | bf16_mfu: 32.06 | epoch: 1 pq: 10 rg: 32 | total time: 37.31m | eta: 24.9m +step 00905/01500 (60.33%) | loss: 2.434947 | lrm: 1.00 | dt: 2499.20ms | tok/sec: 209,782 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 32 | total time: 37.35m | eta: 24.8m +step 00906/01500 (60.40%) | loss: 2.480730 | lrm: 1.00 | dt: 2498.09ms | tok/sec: 209,875 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 32 | total time: 37.40m | eta: 24.8m +step 00907/01500 (60.47%) | loss: 2.504384 | lrm: 1.00 | dt: 2499.45ms | tok/sec: 209,761 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 40 | total time: 37.44m | eta: 24.7m +step 00908/01500 (60.53%) | loss: 2.495713 | lrm: 1.00 | dt: 2505.00ms | tok/sec: 209,296 | bf16_mfu: 31.95 | epoch: 1 pq: 10 rg: 40 | total time: 37.48m | eta: 24.7m +step 00909/01500 (60.60%) | loss: 2.462858 | lrm: 1.00 | dt: 2499.99ms | tok/sec: 209,716 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 40 | total time: 37.52m | eta: 24.7m +step 00910/01500 (60.67%) | loss: 2.435030 | lrm: 1.00 | dt: 2505.34ms | tok/sec: 209,268 | bf16_mfu: 31.95 | epoch: 1 pq: 10 rg: 40 | total time: 37.56m | eta: 24.6m +step 00911/01500 (60.73%) | loss: 2.447943 | lrm: 1.00 | dt: 2502.82ms | tok/sec: 209,478 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 40 | total time: 37.60m | eta: 24.6m +step 00912/01500 (60.80%) | loss: 2.454640 | lrm: 1.00 | dt: 2493.91ms | tok/sec: 210,227 | bf16_mfu: 32.10 | epoch: 1 pq: 10 rg: 40 | total time: 37.65m | eta: 24.5m +step 00913/01500 (60.87%) | loss: 2.445828 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,811 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 40 | total time: 37.69m | eta: 24.5m +step 00914/01500 (60.93%) | loss: 2.422156 | lrm: 1.00 | dt: 2495.24ms | tok/sec: 210,115 | bf16_mfu: 32.08 | epoch: 1 pq: 10 rg: 40 | total time: 37.73m | eta: 24.5m +step 00915/01500 (61.00%) | loss: 2.371485 | lrm: 1.00 | dt: 2498.23ms | tok/sec: 209,863 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 48 | total time: 37.77m | eta: 24.4m +step 00916/01500 (61.07%) | loss: 2.345092 | lrm: 1.00 | dt: 2501.57ms | tok/sec: 209,583 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 48 | total time: 37.81m | eta: 24.4m +step 00917/01500 (61.13%) | loss: 2.322704 | lrm: 1.00 | dt: 2503.62ms | tok/sec: 209,411 | bf16_mfu: 31.97 | epoch: 1 pq: 10 rg: 48 | total time: 37.85m | eta: 24.3m +step 00918/01500 (61.20%) | loss: 2.368564 | lrm: 1.00 | dt: 2499.05ms | tok/sec: 209,794 | bf16_mfu: 32.03 | epoch: 1 pq: 10 rg: 48 | total time: 37.90m | eta: 24.3m +step 00919/01500 (61.27%) | loss: 2.370071 | lrm: 1.00 | dt: 2501.15ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 48 | total time: 37.94m | eta: 24.2m +step 00920/01500 (61.33%) | loss: 2.356302 | lrm: 1.00 | dt: 2502.72ms | tok/sec: 209,487 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 48 | total time: 37.98m | eta: 24.2m +step 00921/01500 (61.40%) | loss: 2.315991 | lrm: 1.00 | dt: 2504.31ms | tok/sec: 209,354 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 48 | total time: 38.02m | eta: 24.2m +step 00922/01500 (61.47%) | loss: 2.319314 | lrm: 1.00 | dt: 2499.49ms | tok/sec: 209,758 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 48 | total time: 38.06m | eta: 24.1m +step 00923/01500 (61.53%) | loss: 2.356224 | lrm: 1.00 | dt: 2499.80ms | tok/sec: 209,731 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 56 | total time: 38.10m | eta: 24.1m +step 00924/01500 (61.60%) | loss: 2.359322 | lrm: 1.00 | dt: 2498.15ms | tok/sec: 209,870 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 56 | total time: 38.15m | eta: 24.0m +step 00925/01500 (61.67%) | loss: 2.411088 | lrm: 1.00 | dt: 2497.91ms | tok/sec: 209,890 | bf16_mfu: 32.04 | epoch: 1 pq: 10 rg: 56 | total time: 38.19m | eta: 24.0m +step 00926/01500 (61.73%) | loss: 2.339008 | lrm: 1.00 | dt: 2499.73ms | tok/sec: 209,738 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 56 | total time: 38.23m | eta: 24.0m +step 00927/01500 (61.80%) | loss: 2.375406 | lrm: 1.00 | dt: 2503.46ms | tok/sec: 209,424 | bf16_mfu: 31.97 | epoch: 1 pq: 10 rg: 56 | total time: 38.27m | eta: 23.9m +step 00928/01500 (61.87%) | loss: 2.395494 | lrm: 1.00 | dt: 2501.10ms | tok/sec: 209,622 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 56 | total time: 38.31m | eta: 23.9m +step 00929/01500 (61.93%) | loss: 2.427484 | lrm: 1.00 | dt: 2500.83ms | tok/sec: 209,645 | bf16_mfu: 32.01 | epoch: 1 pq: 10 rg: 56 | total time: 38.35m | eta: 23.8m +step 00930/01500 (62.00%) | loss: 2.435807 | lrm: 1.00 | dt: 2502.10ms | tok/sec: 209,539 | bf16_mfu: 31.99 | epoch: 1 pq: 10 rg: 56 | total time: 38.40m | eta: 23.8m +step 00931/01500 (62.07%) | loss: 2.426410 | lrm: 1.00 | dt: 2504.83ms | tok/sec: 209,310 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 56 | total time: 38.44m | eta: 23.7m +step 00932/01500 (62.13%) | loss: 2.424321 | lrm: 1.00 | dt: 2494.62ms | tok/sec: 210,167 | bf16_mfu: 32.09 | epoch: 1 pq: 10 rg: 64 | total time: 38.48m | eta: 23.7m +step 00933/01500 (62.20%) | loss: 2.443905 | lrm: 1.00 | dt: 2501.26ms | tok/sec: 209,609 | bf16_mfu: 32.00 | epoch: 1 pq: 10 rg: 64 | total time: 38.52m | eta: 23.7m +step 00934/01500 (62.27%) | loss: 2.477102 | lrm: 1.00 | dt: 2502.26ms | tok/sec: 209,525 | bf16_mfu: 31.99 | epoch: 1 pq: 10 rg: 64 | total time: 38.56m | eta: 23.6m +step 00935/01500 (62.33%) | loss: 2.472756 | lrm: 1.00 | dt: 2503.48ms | tok/sec: 209,424 | bf16_mfu: 31.97 | epoch: 1 pq: 10 rg: 64 | total time: 38.60m | eta: 23.6m +step 00936/01500 (62.40%) | loss: 2.477611 | lrm: 1.00 | dt: 2495.53ms | tok/sec: 210,090 | bf16_mfu: 32.08 | epoch: 1 pq: 10 rg: 64 | total time: 38.65m | eta: 23.5m +step 00937/01500 (62.47%) | loss: 2.486416 | lrm: 1.00 | dt: 2504.99ms | tok/sec: 209,297 | bf16_mfu: 31.95 | epoch: 1 pq: 10 rg: 64 | total time: 38.69m | eta: 23.5m +step 00938/01500 (62.53%) | loss: 2.504975 | lrm: 1.00 | dt: 2502.44ms | tok/sec: 209,510 | bf16_mfu: 31.99 | epoch: 1 pq: 10 rg: 64 | total time: 38.73m | eta: 23.5m +step 00939/01500 (62.60%) | loss: 2.523929 | lrm: 1.00 | dt: 2512.75ms | tok/sec: 208,650 | bf16_mfu: 31.86 | epoch: 1 pq: 10 rg: 64 | total time: 38.77m | eta: 23.4m +step 00940/01500 (62.67%) | loss: 2.517792 | lrm: 1.00 | dt: 2499.65ms | tok/sec: 209,744 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 72 | total time: 38.81m | eta: 23.4m +step 00941/01500 (62.73%) | loss: 2.534914 | lrm: 1.00 | dt: 2499.52ms | tok/sec: 209,755 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 72 | total time: 38.85m | eta: 23.3m +step 00942/01500 (62.80%) | loss: 2.592099 | lrm: 1.00 | dt: 2508.99ms | tok/sec: 208,963 | bf16_mfu: 31.90 | epoch: 1 pq: 10 rg: 72 | total time: 38.90m | eta: 23.3m +step 00943/01500 (62.87%) | loss: 2.642259 | lrm: 1.00 | dt: 2506.59ms | tok/sec: 209,164 | bf16_mfu: 31.93 | epoch: 1 pq: 10 rg: 72 | total time: 38.94m | eta: 23.2m +step 00944/01500 (62.93%) | loss: 2.654288 | lrm: 1.00 | dt: 2500.12ms | tok/sec: 209,704 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 72 | total time: 38.98m | eta: 23.2m +step 00945/01500 (63.00%) | loss: 2.667749 | lrm: 1.00 | dt: 2504.46ms | tok/sec: 209,341 | bf16_mfu: 31.96 | epoch: 1 pq: 10 rg: 72 | total time: 39.02m | eta: 23.2m +step 00946/01500 (63.07%) | loss: 2.651784 | lrm: 1.00 | dt: 2864.18ms | tok/sec: 183,049 | bf16_mfu: 27.95 | epoch: 1 pq: 10 rg: 72 | total time: 39.07m | eta: 23.1m +step 00947/01500 (63.13%) | loss: 2.667027 | lrm: 1.00 | dt: 2502.78ms | tok/sec: 209,482 | bf16_mfu: 31.98 | epoch: 1 pq: 10 rg: 72 | total time: 39.11m | eta: 23.1m +step 00948/01500 (63.20%) | loss: 2.621160 | lrm: 1.00 | dt: 2505.13ms | tok/sec: 209,286 | bf16_mfu: 31.95 | epoch: 1 pq: 10 rg: 80 | total time: 39.15m | eta: 23.0m +step 00949/01500 (63.27%) | loss: 2.607871 | lrm: 1.00 | dt: 2499.78ms | tok/sec: 209,733 | bf16_mfu: 32.02 | epoch: 1 pq: 10 rg: 80 | total time: 39.19m | eta: 23.0m +step 00950/01500 (63.33%) | loss: 2.592276 | lrm: 1.00 | dt: 2497.56ms | tok/sec: 209,919 | bf16_mfu: 32.05 | epoch: 1 pq: 10 rg: 80 | total time: 39.24m | eta: 23.0m +step 00951/01500 (63.40%) | loss: 2.583517 | lrm: 1.00 | dt: 2507.85ms | tok/sec: 209,058 | bf16_mfu: 31.92 | epoch: 1 pq: 10 rg: 80 | total time: 39.28m | eta: 22.9m +step 00952/01500 (63.47%) | loss: 2.591586 | lrm: 1.00 | dt: 2495.67ms | tok/sec: 210,079 | bf16_mfu: 32.07 | epoch: 1 pq: 10 rg: 80 | total time: 39.32m | eta: 22.9m +step 00953/01500 (63.53%) | loss: 2.588482 | lrm: 1.00 | dt: 2496.82ms | tok/sec: 209,982 | bf16_mfu: 32.06 | epoch: 1 pq: 10 rg: 80 | total time: 39.36m | eta: 22.8m +step 00954/01500 (63.60%) | loss: 2.538744 | lrm: 1.00 | dt: 2500.56ms | tok/sec: 209,668 | bf16_mfu: 32.01 | epoch: 1 pq: 10 rg: 80 | total time: 39.40m | eta: 22.8m +step 00955/01500 (63.67%) | loss: 2.597563 | lrm: 1.00 | dt: 2499.85ms | tok/sec: 209,727 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 0 | total time: 39.44m | eta: 22.7m +step 00956/01500 (63.73%) | loss: 2.552908 | lrm: 1.00 | dt: 2500.05ms | tok/sec: 209,710 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 0 | total time: 39.49m | eta: 22.7m +step 00957/01500 (63.80%) | loss: 2.587846 | lrm: 1.00 | dt: 2499.13ms | tok/sec: 209,788 | bf16_mfu: 32.03 | epoch: 1 pq: 11 rg: 0 | total time: 39.53m | eta: 22.7m +step 00958/01500 (63.87%) | loss: 2.517173 | lrm: 1.00 | dt: 2499.85ms | tok/sec: 209,727 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 0 | total time: 39.57m | eta: 22.6m +step 00959/01500 (63.93%) | loss: 2.483435 | lrm: 1.00 | dt: 2500.10ms | tok/sec: 209,707 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 0 | total time: 39.61m | eta: 22.6m +step 00960/01500 (64.00%) | loss: 2.511735 | lrm: 1.00 | dt: 2497.28ms | tok/sec: 209,943 | bf16_mfu: 32.05 | epoch: 1 pq: 11 rg: 0 | total time: 39.65m | eta: 22.5m +step 00961/01500 (64.07%) | loss: 2.485906 | lrm: 1.00 | dt: 2499.28ms | tok/sec: 209,775 | bf16_mfu: 32.03 | epoch: 1 pq: 11 rg: 0 | total time: 39.69m | eta: 22.5m +step 00962/01500 (64.13%) | loss: 2.495403 | lrm: 1.00 | dt: 2500.50ms | tok/sec: 209,673 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 0 | total time: 39.74m | eta: 22.5m +step 00963/01500 (64.20%) | loss: 2.427355 | lrm: 1.00 | dt: 2501.12ms | tok/sec: 209,621 | bf16_mfu: 32.00 | epoch: 1 pq: 11 rg: 0 | total time: 39.78m | eta: 22.4m +step 00964/01500 (64.27%) | loss: 2.373301 | lrm: 1.00 | dt: 2503.56ms | tok/sec: 209,417 | bf16_mfu: 31.97 | epoch: 1 pq: 11 rg: 8 | total time: 39.82m | eta: 22.4m +step 00965/01500 (64.33%) | loss: 2.403538 | lrm: 1.00 | dt: 2497.58ms | tok/sec: 209,918 | bf16_mfu: 32.05 | epoch: 1 pq: 11 rg: 8 | total time: 39.86m | eta: 22.3m +step 00966/01500 (64.40%) | loss: 2.396511 | lrm: 1.00 | dt: 2499.82ms | tok/sec: 209,730 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 8 | total time: 39.90m | eta: 22.3m +step 00967/01500 (64.47%) | loss: 2.421103 | lrm: 1.00 | dt: 2502.89ms | tok/sec: 209,472 | bf16_mfu: 31.98 | epoch: 1 pq: 11 rg: 8 | total time: 39.94m | eta: 22.2m +step 00968/01500 (64.53%) | loss: 2.371453 | lrm: 1.00 | dt: 2506.20ms | tok/sec: 209,196 | bf16_mfu: 31.94 | epoch: 1 pq: 11 rg: 8 | total time: 39.99m | eta: 22.2m +step 00969/01500 (64.60%) | loss: 2.408201 | lrm: 1.00 | dt: 2500.77ms | tok/sec: 209,650 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 8 | total time: 40.03m | eta: 22.2m +step 00970/01500 (64.67%) | loss: 2.475795 | lrm: 1.00 | dt: 2500.10ms | tok/sec: 209,706 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 8 | total time: 40.07m | eta: 22.1m +step 00971/01500 (64.73%) | loss: 2.476792 | lrm: 1.00 | dt: 2498.16ms | tok/sec: 209,870 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 8 | total time: 40.11m | eta: 22.1m +step 00972/01500 (64.80%) | loss: 2.493392 | lrm: 1.00 | dt: 2502.77ms | tok/sec: 209,482 | bf16_mfu: 31.98 | epoch: 1 pq: 11 rg: 16 | total time: 40.15m | eta: 22.0m +step 00973/01500 (64.87%) | loss: 2.530999 | lrm: 1.00 | dt: 2504.69ms | tok/sec: 209,322 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 16 | total time: 40.19m | eta: 22.0m +step 00974/01500 (64.93%) | loss: 2.523076 | lrm: 1.00 | dt: 2498.97ms | tok/sec: 209,801 | bf16_mfu: 32.03 | epoch: 1 pq: 11 rg: 16 | total time: 40.24m | eta: 22.0m +step 00975/01500 (65.00%) | loss: 2.548861 | lrm: 1.00 | dt: 2503.95ms | tok/sec: 209,384 | bf16_mfu: 31.97 | epoch: 1 pq: 11 rg: 16 | total time: 40.28m | eta: 21.9m +step 00976/01500 (65.07%) | loss: 2.541862 | lrm: 1.00 | dt: 2500.11ms | tok/sec: 209,705 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 16 | total time: 40.32m | eta: 21.9m +step 00977/01500 (65.13%) | loss: 2.530597 | lrm: 1.00 | dt: 2501.88ms | tok/sec: 209,557 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 16 | total time: 40.36m | eta: 21.8m +step 00978/01500 (65.20%) | loss: 2.512620 | lrm: 1.00 | dt: 2500.03ms | tok/sec: 209,712 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 16 | total time: 40.40m | eta: 21.8m +step 00979/01500 (65.27%) | loss: 2.513045 | lrm: 1.00 | dt: 2501.07ms | tok/sec: 209,625 | bf16_mfu: 32.00 | epoch: 1 pq: 11 rg: 16 | total time: 40.44m | eta: 21.7m +step 00980/01500 (65.33%) | loss: 2.520401 | lrm: 1.00 | dt: 2500.33ms | tok/sec: 209,687 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 24 | total time: 40.49m | eta: 21.7m +step 00981/01500 (65.40%) | loss: 2.507651 | lrm: 1.00 | dt: 2498.06ms | tok/sec: 209,877 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 24 | total time: 40.53m | eta: 21.7m +step 00982/01500 (65.47%) | loss: 2.498525 | lrm: 1.00 | dt: 2498.41ms | tok/sec: 209,848 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 24 | total time: 40.57m | eta: 21.6m +step 00983/01500 (65.53%) | loss: 2.495252 | lrm: 1.00 | dt: 2504.27ms | tok/sec: 209,357 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 24 | total time: 40.61m | eta: 21.6m +step 00984/01500 (65.60%) | loss: 2.509092 | lrm: 1.00 | dt: 2504.26ms | tok/sec: 209,358 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 24 | total time: 40.65m | eta: 21.5m +step 00985/01500 (65.67%) | loss: 2.470145 | lrm: 1.00 | dt: 2498.52ms | tok/sec: 209,839 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 24 | total time: 40.69m | eta: 21.5m +step 00986/01500 (65.73%) | loss: 2.498084 | lrm: 1.00 | dt: 2504.01ms | tok/sec: 209,379 | bf16_mfu: 31.97 | epoch: 1 pq: 11 rg: 24 | total time: 40.74m | eta: 21.5m +step 00987/01500 (65.80%) | loss: 2.484797 | lrm: 1.00 | dt: 2503.29ms | tok/sec: 209,439 | bf16_mfu: 31.98 | epoch: 1 pq: 11 rg: 24 | total time: 40.78m | eta: 21.4m +step 00988/01500 (65.87%) | loss: 2.476182 | lrm: 1.00 | dt: 2522.53ms | tok/sec: 207,841 | bf16_mfu: 31.73 | epoch: 1 pq: 11 rg: 32 | total time: 40.82m | eta: 21.4m +step 00989/01500 (65.93%) | loss: 2.454756 | lrm: 1.00 | dt: 2502.31ms | tok/sec: 209,521 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 32 | total time: 40.86m | eta: 21.3m +step 00990/01500 (66.00%) | loss: 2.453343 | lrm: 1.00 | dt: 2504.37ms | tok/sec: 209,349 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 32 | total time: 40.90m | eta: 21.3m +step 00991/01500 (66.07%) | loss: 2.431266 | lrm: 1.00 | dt: 2498.48ms | tok/sec: 209,843 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 32 | total time: 40.95m | eta: 21.2m +step 00992/01500 (66.13%) | loss: 2.458795 | lrm: 1.00 | dt: 2502.59ms | tok/sec: 209,498 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 32 | total time: 40.99m | eta: 21.2m +step 00993/01500 (66.20%) | loss: 2.454704 | lrm: 1.00 | dt: 2504.36ms | tok/sec: 209,350 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 32 | total time: 41.03m | eta: 21.2m +step 00994/01500 (66.27%) | loss: 2.408088 | lrm: 1.00 | dt: 2496.91ms | tok/sec: 209,974 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 32 | total time: 41.07m | eta: 21.1m +step 00995/01500 (66.33%) | loss: 2.413461 | lrm: 1.00 | dt: 2498.20ms | tok/sec: 209,866 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 32 | total time: 41.11m | eta: 21.1m +step 00996/01500 (66.40%) | loss: 2.423102 | lrm: 1.00 | dt: 2515.28ms | tok/sec: 208,441 | bf16_mfu: 31.82 | epoch: 1 pq: 11 rg: 40 | total time: 41.15m | eta: 21.0m +step 00997/01500 (66.47%) | loss: 2.420640 | lrm: 1.00 | dt: 2495.56ms | tok/sec: 210,088 | bf16_mfu: 32.08 | epoch: 1 pq: 11 rg: 40 | total time: 41.20m | eta: 21.0m +step 00998/01500 (66.53%) | loss: 2.447795 | lrm: 1.00 | dt: 2500.33ms | tok/sec: 209,687 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 40 | total time: 41.24m | eta: 21.0m +step 00999/01500 (66.60%) | loss: 2.445551 | lrm: 1.00 | dt: 2499.24ms | tok/sec: 209,778 | bf16_mfu: 32.03 | epoch: 1 pq: 11 rg: 40 | total time: 41.28m | eta: 20.9m +Step 01000 | Validation bpb: 0.735113 +Step 01000 | ICL early: 2.6780 late: 2.8650 score: +0.1870 | Induction loss: 9.4661 acc: 0.0879 (random-half: 14.0060) +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au. It is a chemical element with the symbol Au. It is a chemical +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. Today is the 14th day of the month. The 14 +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky, the ocean, and the sky. I love +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +2026-08-10 22:31:14,048 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank6.pt +2026-08-10 22:31:14,754 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank3.pt +2026-08-10 22:31:15,333 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank4.pt +2026-08-10 22:31:17,835 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank5.pt +2026-08-10 22:31:18,694 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank1.pt +2026-08-10 22:31:19,271 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank2.pt +2026-08-10 22:32:18,578 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank7.pt +2026-08-10 22:32:34,385 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_001000.pt +2026-08-10 22:32:34,390 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_001000.json +2026-08-10 22:32:41,351 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001000_rank0.pt +step 01000/01500 (66.67%) | loss: 2.425055 | lrm: 1.00 | dt: 2472.32ms | tok/sec: 212,063 | bf16_mfu: 32.38 | epoch: 1 pq: 11 rg: 40 | total time: 41.32m | eta: 20.9m +step 01001/01500 (66.73%) | loss: 2.437340 | lrm: 1.00 | dt: 2483.08ms | tok/sec: 211,144 | bf16_mfu: 32.24 | epoch: 1 pq: 11 rg: 40 | total time: 41.36m | eta: 20.8m +step 01002/01500 (66.80%) | loss: 2.402506 | lrm: 1.00 | dt: 2487.48ms | tok/sec: 210,770 | bf16_mfu: 32.18 | epoch: 1 pq: 11 rg: 40 | total time: 41.40m | eta: 20.8m +step 01003/01500 (66.87%) | loss: 2.402290 | lrm: 1.00 | dt: 2491.41ms | tok/sec: 210,438 | bf16_mfu: 32.13 | epoch: 1 pq: 11 rg: 40 | total time: 41.44m | eta: 20.7m +step 01004/01500 (66.93%) | loss: 2.383145 | lrm: 1.00 | dt: 2490.94ms | tok/sec: 210,478 | bf16_mfu: 32.13 | epoch: 1 pq: 11 rg: 48 | total time: 41.49m | eta: 20.7m +step 01005/01500 (67.00%) | loss: 2.388188 | lrm: 1.00 | dt: 2496.07ms | tok/sec: 210,045 | bf16_mfu: 32.07 | epoch: 1 pq: 11 rg: 48 | total time: 41.53m | eta: 20.7m +step 01006/01500 (67.07%) | loss: 2.405605 | lrm: 1.00 | dt: 2495.65ms | tok/sec: 210,080 | bf16_mfu: 32.07 | epoch: 1 pq: 11 rg: 48 | total time: 41.57m | eta: 20.6m +step 01007/01500 (67.13%) | loss: 2.419697 | lrm: 1.00 | dt: 2500.89ms | tok/sec: 209,640 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 48 | total time: 41.61m | eta: 20.6m +step 01008/01500 (67.20%) | loss: 2.427633 | lrm: 1.00 | dt: 2499.88ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 48 | total time: 41.65m | eta: 20.5m +step 01009/01500 (67.27%) | loss: 2.466596 | lrm: 1.00 | dt: 2500.15ms | tok/sec: 209,702 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 48 | total time: 41.69m | eta: 20.5m +step 01010/01500 (67.33%) | loss: 2.444322 | lrm: 1.00 | dt: 2496.52ms | tok/sec: 210,007 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 48 | total time: 41.74m | eta: 20.5m +step 01011/01500 (67.40%) | loss: 2.402679 | lrm: 1.00 | dt: 2498.62ms | tok/sec: 209,831 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 48 | total time: 41.78m | eta: 20.4m +step 01012/01500 (67.47%) | loss: 2.347197 | lrm: 1.00 | dt: 2498.42ms | tok/sec: 209,847 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 56 | total time: 41.82m | eta: 20.4m +step 01013/01500 (67.53%) | loss: 2.349528 | lrm: 1.00 | dt: 2498.35ms | tok/sec: 209,853 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 56 | total time: 41.86m | eta: 20.3m +step 01014/01500 (67.60%) | loss: 2.385504 | lrm: 1.00 | dt: 2496.70ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 56 | total time: 41.90m | eta: 20.3m +step 01015/01500 (67.67%) | loss: 2.412254 | lrm: 1.00 | dt: 2505.98ms | tok/sec: 209,214 | bf16_mfu: 31.94 | epoch: 1 pq: 11 rg: 56 | total time: 41.94m | eta: 20.2m +step 01016/01500 (67.73%) | loss: 2.448342 | lrm: 1.00 | dt: 2499.72ms | tok/sec: 209,739 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 56 | total time: 41.99m | eta: 20.2m +step 01017/01500 (67.80%) | loss: 2.467616 | lrm: 1.00 | dt: 2504.20ms | tok/sec: 209,363 | bf16_mfu: 31.96 | epoch: 1 pq: 11 rg: 56 | total time: 42.03m | eta: 20.2m +step 01018/01500 (67.87%) | loss: 2.465448 | lrm: 1.00 | dt: 2494.68ms | tok/sec: 210,162 | bf16_mfu: 32.09 | epoch: 1 pq: 11 rg: 56 | total time: 42.07m | eta: 20.1m +step 01019/01500 (67.93%) | loss: 2.379567 | lrm: 1.00 | dt: 2499.98ms | tok/sec: 209,716 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 56 | total time: 42.11m | eta: 20.1m +step 01020/01500 (68.00%) | loss: 2.391365 | lrm: 1.00 | dt: 2501.41ms | tok/sec: 209,596 | bf16_mfu: 32.00 | epoch: 1 pq: 11 rg: 56 | total time: 42.15m | eta: 20.0m +step 01021/01500 (68.07%) | loss: 2.410615 | lrm: 1.00 | dt: 2496.32ms | tok/sec: 210,024 | bf16_mfu: 32.07 | epoch: 1 pq: 11 rg: 64 | total time: 42.19m | eta: 20.0m +step 01022/01500 (68.13%) | loss: 2.457156 | lrm: 1.00 | dt: 2503.78ms | tok/sec: 209,398 | bf16_mfu: 31.97 | epoch: 1 pq: 11 rg: 64 | total time: 42.24m | eta: 19.9m +step 01023/01500 (68.20%) | loss: 2.501823 | lrm: 1.00 | dt: 2500.79ms | tok/sec: 209,649 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 64 | total time: 42.28m | eta: 19.9m +step 01024/01500 (68.27%) | loss: 2.546373 | lrm: 1.00 | dt: 2496.48ms | tok/sec: 210,010 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 64 | total time: 42.32m | eta: 19.9m +step 01025/01500 (68.33%) | loss: 2.576119 | lrm: 1.00 | dt: 2496.50ms | tok/sec: 210,008 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 64 | total time: 42.36m | eta: 19.8m +step 01026/01500 (68.40%) | loss: 2.607259 | lrm: 1.00 | dt: 2502.13ms | tok/sec: 209,536 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 64 | total time: 42.40m | eta: 19.8m +step 01027/01500 (68.47%) | loss: 2.595531 | lrm: 1.00 | dt: 3246.22ms | tok/sec: 161,507 | bf16_mfu: 24.66 | epoch: 1 pq: 11 rg: 64 | total time: 42.46m | eta: 19.7m +step 01028/01500 (68.53%) | loss: 2.574413 | lrm: 1.00 | dt: 2493.00ms | tok/sec: 210,303 | bf16_mfu: 32.11 | epoch: 1 pq: 11 rg: 64 | total time: 42.50m | eta: 19.7m +step 01029/01500 (68.60%) | loss: 2.571716 | lrm: 1.00 | dt: 2496.43ms | tok/sec: 210,015 | bf16_mfu: 32.06 | epoch: 1 pq: 11 rg: 72 | total time: 42.54m | eta: 19.7m +step 01030/01500 (68.67%) | loss: 2.690301 | lrm: 1.00 | dt: 2499.53ms | tok/sec: 209,754 | bf16_mfu: 32.02 | epoch: 1 pq: 11 rg: 72 | total time: 42.58m | eta: 19.6m +step 01031/01500 (68.73%) | loss: 2.722240 | lrm: 1.00 | dt: 2501.88ms | tok/sec: 209,557 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 72 | total time: 42.62m | eta: 19.6m +step 01032/01500 (68.80%) | loss: 2.717924 | lrm: 1.00 | dt: 2501.72ms | tok/sec: 209,570 | bf16_mfu: 32.00 | epoch: 1 pq: 11 rg: 72 | total time: 42.66m | eta: 19.5m +step 01033/01500 (68.87%) | loss: 2.668257 | lrm: 1.00 | dt: 2496.20ms | tok/sec: 210,034 | bf16_mfu: 32.07 | epoch: 1 pq: 11 rg: 72 | total time: 42.71m | eta: 19.5m +step 01034/01500 (68.93%) | loss: 2.682798 | lrm: 1.00 | dt: 2495.25ms | tok/sec: 210,114 | bf16_mfu: 32.08 | epoch: 1 pq: 11 rg: 72 | total time: 42.75m | eta: 19.5m +step 01035/01500 (69.00%) | loss: 2.604531 | lrm: 1.00 | dt: 2503.97ms | tok/sec: 209,382 | bf16_mfu: 31.97 | epoch: 1 pq: 11 rg: 72 | total time: 42.79m | eta: 19.4m +step 01036/01500 (69.07%) | loss: 2.600583 | lrm: 1.00 | dt: 2498.61ms | tok/sec: 209,832 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 72 | total time: 42.83m | eta: 19.4m +step 01037/01500 (69.13%) | loss: 2.602779 | lrm: 1.00 | dt: 2494.94ms | tok/sec: 210,140 | bf16_mfu: 32.08 | epoch: 1 pq: 11 rg: 80 | total time: 42.87m | eta: 19.3m +step 01038/01500 (69.20%) | loss: 2.547325 | lrm: 1.00 | dt: 2502.10ms | tok/sec: 209,539 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 80 | total time: 42.91m | eta: 19.3m +step 01039/01500 (69.27%) | loss: 2.541426 | lrm: 1.00 | dt: 2507.52ms | tok/sec: 209,086 | bf16_mfu: 31.92 | epoch: 1 pq: 11 rg: 80 | total time: 42.96m | eta: 19.2m +step 01040/01500 (69.33%) | loss: 2.568093 | lrm: 1.00 | dt: 2500.88ms | tok/sec: 209,641 | bf16_mfu: 32.01 | epoch: 1 pq: 11 rg: 80 | total time: 43.00m | eta: 19.2m +step 01041/01500 (69.40%) | loss: 2.592260 | lrm: 1.00 | dt: 2505.36ms | tok/sec: 209,266 | bf16_mfu: 31.95 | epoch: 1 pq: 11 rg: 80 | total time: 43.04m | eta: 19.2m +step 01042/01500 (69.47%) | loss: 2.612483 | lrm: 1.00 | dt: 2501.88ms | tok/sec: 209,557 | bf16_mfu: 31.99 | epoch: 1 pq: 11 rg: 80 | total time: 43.08m | eta: 19.1m +step 01043/01500 (69.53%) | loss: 2.596725 | lrm: 1.00 | dt: 2499.36ms | tok/sec: 209,768 | bf16_mfu: 32.03 | epoch: 1 pq: 11 rg: 80 | total time: 43.12m | eta: 19.1m +step 01044/01500 (69.60%) | loss: 2.567148 | lrm: 1.00 | dt: 2498.40ms | tok/sec: 209,849 | bf16_mfu: 32.04 | epoch: 1 pq: 11 rg: 80 | total time: 43.16m | eta: 19.0m +step 01045/01500 (69.67%) | loss: 2.558828 | lrm: 1.00 | dt: 2500.85ms | tok/sec: 209,644 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 0 | total time: 43.21m | eta: 19.0m +step 01046/01500 (69.73%) | loss: 2.535682 | lrm: 1.00 | dt: 2503.67ms | tok/sec: 209,408 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 0 | total time: 43.25m | eta: 19.0m +step 01047/01500 (69.80%) | loss: 2.538711 | lrm: 1.00 | dt: 2501.11ms | tok/sec: 209,622 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 0 | total time: 43.29m | eta: 18.9m +step 01048/01500 (69.87%) | loss: 2.531904 | lrm: 1.00 | dt: 2497.34ms | tok/sec: 209,938 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 0 | total time: 43.33m | eta: 18.9m +step 01049/01500 (69.93%) | loss: 2.528915 | lrm: 1.00 | dt: 2499.77ms | tok/sec: 209,734 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 0 | total time: 43.37m | eta: 18.8m +step 01050/01500 (70.00%) | loss: 2.543039 | lrm: 1.00 | dt: 2500.74ms | tok/sec: 209,652 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 0 | total time: 43.41m | eta: 18.8m +step 01051/01500 (70.07%) | loss: 2.557352 | lrm: 1.00 | dt: 2504.41ms | tok/sec: 209,346 | bf16_mfu: 31.96 | epoch: 1 pq: 12 rg: 0 | total time: 43.46m | eta: 18.7m +step 01052/01500 (70.13%) | loss: 2.538566 | lrm: 1.00 | dt: 2494.94ms | tok/sec: 210,140 | bf16_mfu: 32.08 | epoch: 1 pq: 12 rg: 0 | total time: 43.50m | eta: 18.7m +step 01053/01500 (70.20%) | loss: 2.524232 | lrm: 1.00 | dt: 2497.16ms | tok/sec: 209,953 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 8 | total time: 43.54m | eta: 18.7m +step 01054/01500 (70.27%) | loss: 2.535897 | lrm: 1.00 | dt: 2502.53ms | tok/sec: 209,503 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 8 | total time: 43.58m | eta: 18.6m +step 01055/01500 (70.33%) | loss: 2.511785 | lrm: 1.00 | dt: 2497.90ms | tok/sec: 209,891 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 8 | total time: 43.62m | eta: 18.6m +step 01056/01500 (70.40%) | loss: 2.540567 | lrm: 1.00 | dt: 2499.79ms | tok/sec: 209,733 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 8 | total time: 43.66m | eta: 18.5m +step 01057/01500 (70.47%) | loss: 2.489501 | lrm: 1.00 | dt: 2500.49ms | tok/sec: 209,673 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 8 | total time: 43.71m | eta: 18.5m +step 01058/01500 (70.53%) | loss: 2.484187 | lrm: 1.00 | dt: 2502.51ms | tok/sec: 209,504 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 8 | total time: 43.75m | eta: 18.5m +step 01059/01500 (70.60%) | loss: 2.487643 | lrm: 1.00 | dt: 2503.50ms | tok/sec: 209,421 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 8 | total time: 43.79m | eta: 18.4m +step 01060/01500 (70.67%) | loss: 2.465699 | lrm: 1.00 | dt: 2500.88ms | tok/sec: 209,641 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 8 | total time: 43.83m | eta: 18.4m +step 01061/01500 (70.73%) | loss: 2.440185 | lrm: 1.00 | dt: 2499.36ms | tok/sec: 209,768 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 16 | total time: 43.87m | eta: 18.3m +step 01062/01500 (70.80%) | loss: 2.421542 | lrm: 1.00 | dt: 2498.22ms | tok/sec: 209,864 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 16 | total time: 43.91m | eta: 18.3m +step 01063/01500 (70.87%) | loss: 2.416919 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 16 | total time: 43.96m | eta: 18.2m +step 01064/01500 (70.93%) | loss: 2.430497 | lrm: 1.00 | dt: 2498.25ms | tok/sec: 209,862 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 16 | total time: 44.00m | eta: 18.2m +step 01065/01500 (71.00%) | loss: 2.405876 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 16 | total time: 44.04m | eta: 18.2m +step 01066/01500 (71.07%) | loss: 2.428573 | lrm: 1.00 | dt: 2498.56ms | tok/sec: 209,836 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 16 | total time: 44.08m | eta: 18.1m +step 01067/01500 (71.13%) | loss: 2.404212 | lrm: 1.00 | dt: 2499.30ms | tok/sec: 209,774 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 16 | total time: 44.12m | eta: 18.1m +step 01068/01500 (71.20%) | loss: 2.394564 | lrm: 1.00 | dt: 2497.63ms | tok/sec: 209,914 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 16 | total time: 44.16m | eta: 18.0m +step 01069/01500 (71.27%) | loss: 2.417578 | lrm: 1.00 | dt: 2501.42ms | tok/sec: 209,596 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 24 | total time: 44.21m | eta: 18.0m +step 01070/01500 (71.33%) | loss: 2.422496 | lrm: 1.00 | dt: 2501.65ms | tok/sec: 209,576 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 24 | total time: 44.25m | eta: 17.9m +step 01071/01500 (71.40%) | loss: 2.426493 | lrm: 1.00 | dt: 2500.84ms | tok/sec: 209,644 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 24 | total time: 44.29m | eta: 17.9m +step 01072/01500 (71.47%) | loss: 2.430037 | lrm: 1.00 | dt: 2502.49ms | tok/sec: 209,506 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 24 | total time: 44.33m | eta: 17.9m +step 01073/01500 (71.53%) | loss: 2.414623 | lrm: 1.00 | dt: 2501.72ms | tok/sec: 209,571 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 24 | total time: 44.37m | eta: 17.8m +step 01074/01500 (71.60%) | loss: 2.452738 | lrm: 1.00 | dt: 2502.09ms | tok/sec: 209,540 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 24 | total time: 44.41m | eta: 17.8m +step 01075/01500 (71.67%) | loss: 2.483931 | lrm: 1.00 | dt: 2504.53ms | tok/sec: 209,336 | bf16_mfu: 31.96 | epoch: 1 pq: 12 rg: 24 | total time: 44.46m | eta: 17.7m +step 01076/01500 (71.73%) | loss: 2.512765 | lrm: 1.00 | dt: 2503.36ms | tok/sec: 209,434 | bf16_mfu: 31.98 | epoch: 1 pq: 12 rg: 24 | total time: 44.50m | eta: 17.7m +step 01077/01500 (71.80%) | loss: 2.489914 | lrm: 1.00 | dt: 2501.77ms | tok/sec: 209,566 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 32 | total time: 44.54m | eta: 17.7m +step 01078/01500 (71.87%) | loss: 2.481657 | lrm: 1.00 | dt: 2502.19ms | tok/sec: 209,531 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 32 | total time: 44.58m | eta: 17.6m +step 01079/01500 (71.93%) | loss: 2.432451 | lrm: 1.00 | dt: 2501.46ms | tok/sec: 209,593 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 32 | total time: 44.62m | eta: 17.6m +step 01080/01500 (72.00%) | loss: 2.459475 | lrm: 1.00 | dt: 2501.02ms | tok/sec: 209,629 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 32 | total time: 44.66m | eta: 17.5m +step 01081/01500 (72.07%) | loss: 2.503668 | lrm: 1.00 | dt: 2501.72ms | tok/sec: 209,570 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 32 | total time: 44.71m | eta: 17.5m +step 01082/01500 (72.13%) | loss: 2.481614 | lrm: 1.00 | dt: 2500.24ms | tok/sec: 209,694 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 32 | total time: 44.75m | eta: 17.4m +step 01083/01500 (72.20%) | loss: 2.539859 | lrm: 1.00 | dt: 2499.89ms | tok/sec: 209,724 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 32 | total time: 44.79m | eta: 17.4m +step 01084/01500 (72.27%) | loss: 2.565516 | lrm: 1.00 | dt: 2501.42ms | tok/sec: 209,596 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 32 | total time: 44.83m | eta: 17.4m +step 01085/01500 (72.33%) | loss: 2.591217 | lrm: 1.00 | dt: 2497.61ms | tok/sec: 209,915 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 40 | total time: 44.87m | eta: 17.3m +step 01086/01500 (72.40%) | loss: 2.577020 | lrm: 1.00 | dt: 2499.82ms | tok/sec: 209,730 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 40 | total time: 44.91m | eta: 17.3m +step 01087/01500 (72.47%) | loss: 2.575139 | lrm: 1.00 | dt: 2498.06ms | tok/sec: 209,878 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 40 | total time: 44.96m | eta: 17.2m +step 01088/01500 (72.53%) | loss: 2.565641 | lrm: 1.00 | dt: 2499.84ms | tok/sec: 209,728 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 40 | total time: 45.00m | eta: 17.2m +step 01089/01500 (72.60%) | loss: 2.549572 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 40 | total time: 45.04m | eta: 17.2m +step 01090/01500 (72.67%) | loss: 2.563537 | lrm: 1.00 | dt: 2499.82ms | tok/sec: 209,730 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 40 | total time: 45.08m | eta: 17.1m +step 01091/01500 (72.73%) | loss: 2.590664 | lrm: 1.00 | dt: 2498.18ms | tok/sec: 209,868 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 40 | total time: 45.12m | eta: 17.1m +step 01092/01500 (72.80%) | loss: 2.578195 | lrm: 1.00 | dt: 2504.73ms | tok/sec: 209,319 | bf16_mfu: 31.96 | epoch: 1 pq: 12 rg: 40 | total time: 45.16m | eta: 17.0m +step 01093/01500 (72.87%) | loss: 2.571242 | lrm: 1.00 | dt: 2502.18ms | tok/sec: 209,532 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 48 | total time: 45.21m | eta: 17.0m +step 01094/01500 (72.93%) | loss: 2.579990 | lrm: 1.00 | dt: 2501.77ms | tok/sec: 209,567 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 48 | total time: 45.25m | eta: 16.9m +step 01095/01500 (73.00%) | loss: 2.537517 | lrm: 1.00 | dt: 2505.12ms | tok/sec: 209,286 | bf16_mfu: 31.95 | epoch: 1 pq: 12 rg: 48 | total time: 45.29m | eta: 16.9m +step 01096/01500 (73.07%) | loss: 2.553105 | lrm: 1.00 | dt: 2500.22ms | tok/sec: 209,696 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 48 | total time: 45.33m | eta: 16.9m +step 01097/01500 (73.13%) | loss: 2.558843 | lrm: 1.00 | dt: 2501.56ms | tok/sec: 209,584 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 48 | total time: 45.37m | eta: 16.8m +step 01098/01500 (73.20%) | loss: 2.570094 | lrm: 1.00 | dt: 2499.03ms | tok/sec: 209,796 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 48 | total time: 45.42m | eta: 16.8m +step 01099/01500 (73.27%) | loss: 2.582346 | lrm: 1.00 | dt: 2503.95ms | tok/sec: 209,384 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 48 | total time: 45.46m | eta: 16.7m +Step 01100 | Validation bpb: 0.732576 +Step 01100 | ICL early: 2.6734 late: 2.8559 score: +0.1825 | Induction loss: 9.4218 acc: 0.0877 (random-half: 13.9912) +step 01100/01500 (73.33%) | loss: 2.586440 | lrm: 1.00 | dt: 2496.63ms | tok/sec: 209,997 | bf16_mfu: 32.06 | epoch: 1 pq: 12 rg: 48 | total time: 45.50m | eta: 16.7m +step 01101/01500 (73.40%) | loss: 2.580091 | lrm: 1.00 | dt: 2499.67ms | tok/sec: 209,742 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 56 | total time: 45.54m | eta: 16.7m +step 01102/01500 (73.47%) | loss: 2.545139 | lrm: 1.00 | dt: 2499.03ms | tok/sec: 209,796 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 56 | total time: 45.58m | eta: 16.6m +step 01103/01500 (73.53%) | loss: 2.540063 | lrm: 1.00 | dt: 2500.63ms | tok/sec: 209,661 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 56 | total time: 45.62m | eta: 16.6m +step 01104/01500 (73.60%) | loss: 2.527436 | lrm: 1.00 | dt: 2502.48ms | tok/sec: 209,506 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 56 | total time: 45.67m | eta: 16.5m +step 01105/01500 (73.67%) | loss: 2.461445 | lrm: 1.00 | dt: 2500.80ms | tok/sec: 209,647 | bf16_mfu: 32.01 | epoch: 1 pq: 12 rg: 56 | total time: 45.71m | eta: 16.5m +step 01106/01500 (73.73%) | loss: 2.489434 | lrm: 1.00 | dt: 2504.12ms | tok/sec: 209,370 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 56 | total time: 45.75m | eta: 16.4m +step 01107/01500 (73.80%) | loss: 2.519811 | lrm: 1.00 | dt: 2499.64ms | tok/sec: 209,745 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 56 | total time: 45.79m | eta: 16.4m +step 01108/01500 (73.87%) | loss: 2.539433 | lrm: 1.00 | dt: 3348.74ms | tok/sec: 156,562 | bf16_mfu: 23.90 | epoch: 1 pq: 12 rg: 56 | total time: 45.85m | eta: 16.4m +step 01109/01500 (73.93%) | loss: 2.525969 | lrm: 1.00 | dt: 2499.52ms | tok/sec: 209,755 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 64 | total time: 45.89m | eta: 16.3m +step 01110/01500 (74.00%) | loss: 2.483961 | lrm: 1.00 | dt: 2501.22ms | tok/sec: 209,613 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 64 | total time: 45.93m | eta: 16.3m +step 01111/01500 (74.07%) | loss: 2.447209 | lrm: 1.00 | dt: 2508.64ms | tok/sec: 208,992 | bf16_mfu: 31.91 | epoch: 1 pq: 12 rg: 64 | total time: 45.97m | eta: 16.2m +step 01112/01500 (74.13%) | loss: 2.446468 | lrm: 1.00 | dt: 2502.01ms | tok/sec: 209,546 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 64 | total time: 46.01m | eta: 16.2m +step 01113/01500 (74.20%) | loss: 2.491553 | lrm: 1.00 | dt: 2506.43ms | tok/sec: 209,177 | bf16_mfu: 31.94 | epoch: 1 pq: 12 rg: 64 | total time: 46.05m | eta: 16.2m +step 01114/01500 (74.27%) | loss: 2.459502 | lrm: 1.00 | dt: 2497.85ms | tok/sec: 209,895 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 64 | total time: 46.10m | eta: 16.1m +step 01115/01500 (74.33%) | loss: 2.445792 | lrm: 1.00 | dt: 2501.11ms | tok/sec: 209,622 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 64 | total time: 46.14m | eta: 16.1m +step 01116/01500 (74.40%) | loss: 2.443582 | lrm: 1.00 | dt: 2502.09ms | tok/sec: 209,540 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 64 | total time: 46.18m | eta: 16.0m +step 01117/01500 (74.47%) | loss: 2.398858 | lrm: 1.00 | dt: 2497.99ms | tok/sec: 209,883 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 72 | total time: 46.22m | eta: 16.0m +step 01118/01500 (74.53%) | loss: 2.435793 | lrm: 1.00 | dt: 2502.90ms | tok/sec: 209,472 | bf16_mfu: 31.98 | epoch: 1 pq: 12 rg: 72 | total time: 46.26m | eta: 15.9m +step 01119/01500 (74.60%) | loss: 2.450739 | lrm: 1.00 | dt: 2501.60ms | tok/sec: 209,580 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 72 | total time: 46.30m | eta: 15.9m +step 01120/01500 (74.67%) | loss: 2.453562 | lrm: 1.00 | dt: 2501.61ms | tok/sec: 209,580 | bf16_mfu: 32.00 | epoch: 1 pq: 12 rg: 72 | total time: 46.35m | eta: 15.9m +step 01121/01500 (74.73%) | loss: 2.458797 | lrm: 1.00 | dt: 2503.79ms | tok/sec: 209,397 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 72 | total time: 46.39m | eta: 15.8m +step 01122/01500 (74.80%) | loss: 2.506607 | lrm: 1.00 | dt: 2503.03ms | tok/sec: 209,461 | bf16_mfu: 31.98 | epoch: 1 pq: 12 rg: 72 | total time: 46.43m | eta: 15.8m +step 01123/01500 (74.87%) | loss: 2.531235 | lrm: 1.00 | dt: 2499.23ms | tok/sec: 209,779 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 72 | total time: 46.47m | eta: 15.7m +step 01124/01500 (74.93%) | loss: 2.536931 | lrm: 1.00 | dt: 2506.69ms | tok/sec: 209,155 | bf16_mfu: 31.93 | epoch: 1 pq: 12 rg: 72 | total time: 46.51m | eta: 15.7m +step 01125/01500 (75.00%) | loss: 2.524225 | lrm: 1.00 | dt: 2503.65ms | tok/sec: 209,409 | bf16_mfu: 31.97 | epoch: 1 pq: 12 rg: 80 | total time: 46.56m | eta: 15.7m +step 01126/01500 (75.07%) | loss: 2.552255 | lrm: 1.00 | dt: 2498.45ms | tok/sec: 209,845 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 80 | total time: 46.60m | eta: 15.6m +step 01127/01500 (75.13%) | loss: 2.552342 | lrm: 1.00 | dt: 2498.11ms | tok/sec: 209,873 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 80 | total time: 46.64m | eta: 15.6m +step 01128/01500 (75.20%) | loss: 2.509837 | lrm: 1.00 | dt: 2502.03ms | tok/sec: 209,544 | bf16_mfu: 31.99 | epoch: 1 pq: 12 rg: 80 | total time: 46.68m | eta: 15.5m +step 01129/01500 (75.27%) | loss: 2.483425 | lrm: 1.00 | dt: 2499.42ms | tok/sec: 209,764 | bf16_mfu: 32.03 | epoch: 1 pq: 12 rg: 80 | total time: 46.72m | eta: 15.5m +step 01130/01500 (75.33%) | loss: 2.458530 | lrm: 1.00 | dt: 2498.58ms | tok/sec: 209,834 | bf16_mfu: 32.04 | epoch: 1 pq: 12 rg: 80 | total time: 46.76m | eta: 15.4m +step 01131/01500 (75.40%) | loss: 2.472561 | lrm: 1.00 | dt: 2497.88ms | tok/sec: 209,893 | bf16_mfu: 32.05 | epoch: 1 pq: 12 rg: 80 | total time: 46.80m | eta: 15.4m +step 01132/01500 (75.47%) | loss: 2.425508 | lrm: 1.00 | dt: 2499.91ms | tok/sec: 209,722 | bf16_mfu: 32.02 | epoch: 1 pq: 12 rg: 80 | total time: 46.85m | eta: 15.4m +step 01133/01500 (75.53%) | loss: 2.371697 | lrm: 1.00 | dt: 2494.87ms | tok/sec: 210,146 | bf16_mfu: 32.08 | epoch: 1 pq: 13 rg: 0 | total time: 46.89m | eta: 15.3m +step 01134/01500 (75.60%) | loss: 2.387674 | lrm: 1.00 | dt: 2495.54ms | tok/sec: 210,090 | bf16_mfu: 32.08 | epoch: 1 pq: 13 rg: 0 | total time: 46.93m | eta: 15.3m +step 01135/01500 (75.67%) | loss: 2.403613 | lrm: 1.00 | dt: 2497.57ms | tok/sec: 209,919 | bf16_mfu: 32.05 | epoch: 1 pq: 13 rg: 0 | total time: 46.97m | eta: 15.2m +step 01136/01500 (75.73%) | loss: 2.420161 | lrm: 1.00 | dt: 2504.45ms | tok/sec: 209,342 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 0 | total time: 47.01m | eta: 15.2m +step 01137/01500 (75.80%) | loss: 2.432826 | lrm: 1.00 | dt: 2500.64ms | tok/sec: 209,661 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 0 | total time: 47.05m | eta: 15.2m +step 01138/01500 (75.87%) | loss: 2.469571 | lrm: 1.00 | dt: 2499.01ms | tok/sec: 209,798 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 0 | total time: 47.10m | eta: 15.1m +step 01139/01500 (75.93%) | loss: 2.512893 | lrm: 1.00 | dt: 2501.99ms | tok/sec: 209,548 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 0 | total time: 47.14m | eta: 15.1m +step 01140/01500 (76.00%) | loss: 2.490199 | lrm: 1.00 | dt: 2499.42ms | tok/sec: 209,763 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 0 | total time: 47.18m | eta: 15.0m +step 01141/01500 (76.07%) | loss: 2.503078 | lrm: 1.00 | dt: 2498.09ms | tok/sec: 209,875 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 8 | total time: 47.22m | eta: 15.0m +step 01142/01500 (76.13%) | loss: 2.508764 | lrm: 1.00 | dt: 2496.40ms | tok/sec: 210,017 | bf16_mfu: 32.06 | epoch: 1 pq: 13 rg: 8 | total time: 47.26m | eta: 14.9m +step 01143/01500 (76.20%) | loss: 2.497010 | lrm: 1.00 | dt: 2496.84ms | tok/sec: 209,980 | bf16_mfu: 32.06 | epoch: 1 pq: 13 rg: 8 | total time: 47.30m | eta: 14.9m +step 01144/01500 (76.27%) | loss: 2.474529 | lrm: 1.00 | dt: 2502.73ms | tok/sec: 209,486 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 8 | total time: 47.35m | eta: 14.9m +step 01145/01500 (76.33%) | loss: 2.409986 | lrm: 1.00 | dt: 2502.98ms | tok/sec: 209,465 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 8 | total time: 47.39m | eta: 14.8m +step 01146/01500 (76.40%) | loss: 2.436207 | lrm: 1.00 | dt: 2499.66ms | tok/sec: 209,743 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 8 | total time: 47.43m | eta: 14.8m +step 01147/01500 (76.47%) | loss: 2.360981 | lrm: 1.00 | dt: 2500.45ms | tok/sec: 209,677 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 8 | total time: 47.47m | eta: 14.7m +step 01148/01500 (76.53%) | loss: 2.366878 | lrm: 1.00 | dt: 2498.45ms | tok/sec: 209,845 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 8 | total time: 47.51m | eta: 14.7m +step 01149/01500 (76.60%) | loss: 2.446126 | lrm: 1.00 | dt: 2500.18ms | tok/sec: 209,699 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 16 | total time: 47.55m | eta: 14.7m +step 01150/01500 (76.67%) | loss: 2.529810 | lrm: 1.00 | dt: 2496.55ms | tok/sec: 210,004 | bf16_mfu: 32.06 | epoch: 1 pq: 13 rg: 16 | total time: 47.60m | eta: 14.6m +step 01151/01500 (76.73%) | loss: 2.563946 | lrm: 1.00 | dt: 2503.01ms | tok/sec: 209,462 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 16 | total time: 47.64m | eta: 14.6m +step 01152/01500 (76.80%) | loss: 2.584806 | lrm: 1.00 | dt: 2500.47ms | tok/sec: 209,675 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 16 | total time: 47.68m | eta: 14.5m +step 01153/01500 (76.87%) | loss: 2.592696 | lrm: 1.00 | dt: 2499.77ms | tok/sec: 209,734 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 16 | total time: 47.72m | eta: 14.5m +step 01154/01500 (76.93%) | loss: 2.579082 | lrm: 1.00 | dt: 2498.89ms | tok/sec: 209,808 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 16 | total time: 47.76m | eta: 14.4m +step 01155/01500 (77.00%) | loss: 2.558637 | lrm: 1.00 | dt: 2501.73ms | tok/sec: 209,570 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 16 | total time: 47.80m | eta: 14.4m +step 01156/01500 (77.07%) | loss: 2.556868 | lrm: 1.00 | dt: 2503.23ms | tok/sec: 209,444 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 16 | total time: 47.85m | eta: 14.4m +step 01157/01500 (77.13%) | loss: 2.561193 | lrm: 1.00 | dt: 2501.57ms | tok/sec: 209,583 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 16 | total time: 47.89m | eta: 14.3m +step 01158/01500 (77.20%) | loss: 2.567330 | lrm: 1.00 | dt: 2499.27ms | tok/sec: 209,776 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 24 | total time: 47.93m | eta: 14.3m +step 01159/01500 (77.27%) | loss: 2.534040 | lrm: 1.00 | dt: 2502.83ms | tok/sec: 209,477 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 24 | total time: 47.97m | eta: 14.2m +step 01160/01500 (77.33%) | loss: 2.520394 | lrm: 1.00 | dt: 2502.64ms | tok/sec: 209,493 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 24 | total time: 48.01m | eta: 14.2m +step 01161/01500 (77.40%) | loss: 2.539431 | lrm: 1.00 | dt: 2499.65ms | tok/sec: 209,744 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 24 | total time: 48.05m | eta: 14.2m +step 01162/01500 (77.47%) | loss: 2.542988 | lrm: 1.00 | dt: 2504.93ms | tok/sec: 209,302 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 24 | total time: 48.10m | eta: 14.1m +step 01163/01500 (77.53%) | loss: 2.557495 | lrm: 1.00 | dt: 2500.02ms | tok/sec: 209,713 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 24 | total time: 48.14m | eta: 14.1m +step 01164/01500 (77.60%) | loss: 2.583253 | lrm: 1.00 | dt: 2499.44ms | tok/sec: 209,762 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 24 | total time: 48.18m | eta: 14.0m +step 01165/01500 (77.67%) | loss: 2.549616 | lrm: 1.00 | dt: 2496.71ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 13 rg: 24 | total time: 48.22m | eta: 14.0m +step 01166/01500 (77.73%) | loss: 2.587962 | lrm: 1.00 | dt: 2498.82ms | tok/sec: 209,813 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 32 | total time: 48.26m | eta: 13.9m +step 01167/01500 (77.80%) | loss: 2.556298 | lrm: 1.00 | dt: 2501.40ms | tok/sec: 209,598 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 32 | total time: 48.30m | eta: 13.9m +step 01168/01500 (77.87%) | loss: 2.576640 | lrm: 1.00 | dt: 2496.13ms | tok/sec: 210,040 | bf16_mfu: 32.07 | epoch: 1 pq: 13 rg: 32 | total time: 48.35m | eta: 13.9m +step 01169/01500 (77.93%) | loss: 2.577357 | lrm: 1.00 | dt: 2501.96ms | tok/sec: 209,550 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 32 | total time: 48.39m | eta: 13.8m +step 01170/01500 (78.00%) | loss: 2.579842 | lrm: 1.00 | dt: 2506.18ms | tok/sec: 209,197 | bf16_mfu: 31.94 | epoch: 1 pq: 13 rg: 32 | total time: 48.43m | eta: 13.8m +step 01171/01500 (78.07%) | loss: 2.553053 | lrm: 1.00 | dt: 2501.56ms | tok/sec: 209,584 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 32 | total time: 48.47m | eta: 13.7m +step 01172/01500 (78.13%) | loss: 2.505223 | lrm: 1.00 | dt: 2500.23ms | tok/sec: 209,696 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 32 | total time: 48.51m | eta: 13.7m +step 01173/01500 (78.20%) | loss: 2.533689 | lrm: 1.00 | dt: 2501.21ms | tok/sec: 209,614 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 32 | total time: 48.56m | eta: 13.7m +step 01174/01500 (78.27%) | loss: 2.529751 | lrm: 1.00 | dt: 2504.64ms | tok/sec: 209,326 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 40 | total time: 48.60m | eta: 13.6m +step 01175/01500 (78.33%) | loss: 2.530845 | lrm: 1.00 | dt: 2502.42ms | tok/sec: 209,512 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 40 | total time: 48.64m | eta: 13.6m +step 01176/01500 (78.40%) | loss: 2.569716 | lrm: 1.00 | dt: 2498.02ms | tok/sec: 209,881 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 40 | total time: 48.68m | eta: 13.5m +step 01177/01500 (78.47%) | loss: 2.557234 | lrm: 1.00 | dt: 2504.77ms | tok/sec: 209,316 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 40 | total time: 48.72m | eta: 13.5m +step 01178/01500 (78.53%) | loss: 2.550491 | lrm: 1.00 | dt: 2505.66ms | tok/sec: 209,241 | bf16_mfu: 31.95 | epoch: 1 pq: 13 rg: 40 | total time: 48.76m | eta: 13.4m +step 01179/01500 (78.60%) | loss: 2.539480 | lrm: 1.00 | dt: 2504.82ms | tok/sec: 209,311 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 40 | total time: 48.81m | eta: 13.4m +step 01180/01500 (78.67%) | loss: 2.553053 | lrm: 1.00 | dt: 2498.57ms | tok/sec: 209,835 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 40 | total time: 48.85m | eta: 13.4m +step 01181/01500 (78.73%) | loss: 2.548203 | lrm: 1.00 | dt: 2503.52ms | tok/sec: 209,420 | bf16_mfu: 31.97 | epoch: 1 pq: 13 rg: 40 | total time: 48.89m | eta: 13.3m +step 01182/01500 (78.80%) | loss: 2.497362 | lrm: 1.00 | dt: 2500.40ms | tok/sec: 209,681 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 48 | total time: 48.93m | eta: 13.3m +step 01183/01500 (78.87%) | loss: 2.533041 | lrm: 1.00 | dt: 2500.14ms | tok/sec: 209,703 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 48 | total time: 48.97m | eta: 13.2m +step 01184/01500 (78.93%) | loss: 2.502415 | lrm: 1.00 | dt: 2499.05ms | tok/sec: 209,794 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 48 | total time: 49.01m | eta: 13.2m +step 01185/01500 (79.00%) | loss: 2.432266 | lrm: 1.00 | dt: 2499.68ms | tok/sec: 209,742 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 48 | total time: 49.06m | eta: 13.2m +step 01186/01500 (79.07%) | loss: 2.429734 | lrm: 1.00 | dt: 2499.95ms | tok/sec: 209,719 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 48 | total time: 49.10m | eta: 13.1m +step 01187/01500 (79.13%) | loss: 2.438783 | lrm: 1.00 | dt: 2503.01ms | tok/sec: 209,462 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 48 | total time: 49.14m | eta: 13.1m +step 01188/01500 (79.20%) | loss: 2.412814 | lrm: 1.00 | dt: 2503.67ms | tok/sec: 209,407 | bf16_mfu: 31.97 | epoch: 1 pq: 13 rg: 48 | total time: 49.18m | eta: 13.0m +step 01189/01500 (79.27%) | loss: 2.417956 | lrm: 1.00 | dt: 3166.04ms | tok/sec: 165,597 | bf16_mfu: 25.28 | epoch: 1 pq: 13 rg: 48 | total time: 49.23m | eta: 13.0m +step 01190/01500 (79.33%) | loss: 2.391732 | lrm: 1.00 | dt: 2495.70ms | tok/sec: 210,076 | bf16_mfu: 32.07 | epoch: 1 pq: 13 rg: 56 | total time: 49.27m | eta: 12.9m +step 01191/01500 (79.40%) | loss: 2.413218 | lrm: 1.00 | dt: 2500.86ms | tok/sec: 209,643 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 56 | total time: 49.32m | eta: 12.9m +step 01192/01500 (79.47%) | loss: 2.409355 | lrm: 1.00 | dt: 2498.83ms | tok/sec: 209,813 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 56 | total time: 49.36m | eta: 12.9m +step 01193/01500 (79.53%) | loss: 2.403850 | lrm: 1.00 | dt: 2502.16ms | tok/sec: 209,534 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 56 | total time: 49.40m | eta: 12.8m +step 01194/01500 (79.60%) | loss: 2.383438 | lrm: 1.00 | dt: 2501.48ms | tok/sec: 209,590 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 56 | total time: 49.44m | eta: 12.8m +step 01195/01500 (79.67%) | loss: 2.384559 | lrm: 1.00 | dt: 2499.75ms | tok/sec: 209,735 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 56 | total time: 49.48m | eta: 12.7m +step 01196/01500 (79.73%) | loss: 2.327422 | lrm: 1.00 | dt: 2497.07ms | tok/sec: 209,961 | bf16_mfu: 32.06 | epoch: 1 pq: 13 rg: 56 | total time: 49.52m | eta: 12.7m +step 01197/01500 (79.80%) | loss: 2.384903 | lrm: 1.00 | dt: 2504.17ms | tok/sec: 209,365 | bf16_mfu: 31.96 | epoch: 1 pq: 13 rg: 56 | total time: 49.57m | eta: 12.7m +step 01198/01500 (79.87%) | loss: 2.424343 | lrm: 1.00 | dt: 2494.40ms | tok/sec: 210,186 | bf16_mfu: 32.09 | epoch: 1 pq: 13 rg: 64 | total time: 49.61m | eta: 12.6m +step 01199/01500 (79.93%) | loss: 2.430876 | lrm: 1.00 | dt: 2497.48ms | tok/sec: 209,926 | bf16_mfu: 32.05 | epoch: 1 pq: 13 rg: 64 | total time: 49.65m | eta: 12.6m +Step 01200 | Validation bpb: 0.730429 +Step 01200 | ICL early: 2.6633 late: 2.8423 score: +0.1790 | Induction loss: 9.2954 acc: 0.0939 (random-half: 14.1187) +step 01200/01500 (80.00%) | loss: 2.440996 | lrm: 1.00 | dt: 2493.94ms | tok/sec: 210,224 | bf16_mfu: 32.10 | epoch: 1 pq: 13 rg: 64 | total time: 49.69m | eta: 12.5m +step 01201/01500 (80.07%) | loss: 2.432617 | lrm: 1.00 | dt: 2497.17ms | tok/sec: 209,952 | bf16_mfu: 32.05 | epoch: 1 pq: 13 rg: 64 | total time: 49.73m | eta: 12.5m +step 01202/01500 (80.13%) | loss: 2.445783 | lrm: 1.00 | dt: 2498.38ms | tok/sec: 209,851 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 64 | total time: 49.77m | eta: 12.4m +step 01203/01500 (80.20%) | loss: 2.470045 | lrm: 1.00 | dt: 2500.16ms | tok/sec: 209,702 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 64 | total time: 49.82m | eta: 12.4m +step 01204/01500 (80.27%) | loss: 2.446939 | lrm: 1.00 | dt: 2498.92ms | tok/sec: 209,806 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 64 | total time: 49.86m | eta: 12.4m +step 01205/01500 (80.33%) | loss: 2.424146 | lrm: 1.00 | dt: 2500.20ms | tok/sec: 209,698 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 64 | total time: 49.90m | eta: 12.3m +step 01206/01500 (80.40%) | loss: 2.387269 | lrm: 1.00 | dt: 2503.97ms | tok/sec: 209,382 | bf16_mfu: 31.97 | epoch: 1 pq: 13 rg: 72 | total time: 49.94m | eta: 12.3m +step 01207/01500 (80.47%) | loss: 2.356603 | lrm: 1.00 | dt: 2503.11ms | tok/sec: 209,454 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 72 | total time: 49.98m | eta: 12.2m +step 01208/01500 (80.53%) | loss: 2.381411 | lrm: 1.00 | dt: 2502.20ms | tok/sec: 209,531 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 72 | total time: 50.02m | eta: 12.2m +step 01209/01500 (80.60%) | loss: 2.398901 | lrm: 1.00 | dt: 2498.46ms | tok/sec: 209,844 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 72 | total time: 50.07m | eta: 12.2m +step 01210/01500 (80.67%) | loss: 2.424929 | lrm: 1.00 | dt: 2502.33ms | tok/sec: 209,519 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 72 | total time: 50.11m | eta: 12.1m +step 01211/01500 (80.73%) | loss: 2.432458 | lrm: 1.00 | dt: 2502.75ms | tok/sec: 209,484 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 72 | total time: 50.15m | eta: 12.1m +step 01212/01500 (80.80%) | loss: 2.416121 | lrm: 1.00 | dt: 2500.12ms | tok/sec: 209,704 | bf16_mfu: 32.02 | epoch: 1 pq: 13 rg: 72 | total time: 50.19m | eta: 12.0m +step 01213/01500 (80.87%) | loss: 2.403905 | lrm: 1.00 | dt: 2502.53ms | tok/sec: 209,503 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 72 | total time: 50.23m | eta: 12.0m +step 01214/01500 (80.93%) | loss: 2.445069 | lrm: 1.00 | dt: 2499.27ms | tok/sec: 209,776 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 80 | total time: 50.27m | eta: 11.9m +step 01215/01500 (81.00%) | loss: 2.472159 | lrm: 1.00 | dt: 2499.01ms | tok/sec: 209,798 | bf16_mfu: 32.03 | epoch: 1 pq: 13 rg: 80 | total time: 50.32m | eta: 11.9m +step 01216/01500 (81.07%) | loss: 2.488684 | lrm: 1.00 | dt: 2502.64ms | tok/sec: 209,493 | bf16_mfu: 31.98 | epoch: 1 pq: 13 rg: 80 | total time: 50.36m | eta: 11.9m +step 01217/01500 (81.13%) | loss: 2.488064 | lrm: 1.00 | dt: 2500.84ms | tok/sec: 209,644 | bf16_mfu: 32.01 | epoch: 1 pq: 13 rg: 80 | total time: 50.40m | eta: 11.8m +step 01218/01500 (81.20%) | loss: 2.464704 | lrm: 1.00 | dt: 2502.59ms | tok/sec: 209,497 | bf16_mfu: 31.99 | epoch: 1 pq: 13 rg: 80 | total time: 50.44m | eta: 11.8m +step 01219/01500 (81.27%) | loss: 2.457899 | lrm: 1.00 | dt: 2498.21ms | tok/sec: 209,865 | bf16_mfu: 32.04 | epoch: 1 pq: 13 rg: 80 | total time: 50.48m | eta: 11.7m +step 01220/01500 (81.33%) | loss: 2.463797 | lrm: 1.00 | dt: 2501.32ms | tok/sec: 209,604 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 80 | total time: 50.52m | eta: 11.7m +step 01221/01500 (81.40%) | loss: 2.421569 | lrm: 1.00 | dt: 2501.07ms | tok/sec: 209,625 | bf16_mfu: 32.00 | epoch: 1 pq: 13 rg: 80 | total time: 50.57m | eta: 11.6m +step 01222/01500 (81.47%) | loss: 2.387758 | lrm: 1.00 | dt: 2501.59ms | tok/sec: 209,582 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 0 | total time: 50.61m | eta: 11.6m +step 01223/01500 (81.53%) | loss: 2.403879 | lrm: 1.00 | dt: 2502.32ms | tok/sec: 209,521 | bf16_mfu: 31.99 | epoch: 1 pq: 14 rg: 0 | total time: 50.65m | eta: 11.6m +step 01224/01500 (81.60%) | loss: 2.443035 | lrm: 1.00 | dt: 2496.38ms | tok/sec: 210,019 | bf16_mfu: 32.06 | epoch: 1 pq: 14 rg: 0 | total time: 50.69m | eta: 11.5m +step 01225/01500 (81.67%) | loss: 2.418260 | lrm: 1.00 | dt: 2499.23ms | tok/sec: 209,779 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 0 | total time: 50.73m | eta: 11.5m +step 01226/01500 (81.73%) | loss: 2.448829 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 14 rg: 0 | total time: 50.77m | eta: 11.4m +step 01227/01500 (81.80%) | loss: 2.460668 | lrm: 1.00 | dt: 2503.39ms | tok/sec: 209,431 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 0 | total time: 50.82m | eta: 11.4m +step 01228/01500 (81.87%) | loss: 2.476715 | lrm: 1.00 | dt: 2501.51ms | tok/sec: 209,588 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 0 | total time: 50.86m | eta: 11.4m +step 01229/01500 (81.93%) | loss: 2.423908 | lrm: 1.00 | dt: 2500.45ms | tok/sec: 209,677 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 0 | total time: 50.90m | eta: 11.3m +step 01230/01500 (82.00%) | loss: 2.413463 | lrm: 1.00 | dt: 2501.00ms | tok/sec: 209,631 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 8 | total time: 50.94m | eta: 11.3m +step 01231/01500 (82.07%) | loss: 2.394710 | lrm: 1.00 | dt: 2504.12ms | tok/sec: 209,369 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 8 | total time: 50.98m | eta: 11.2m +step 01232/01500 (82.13%) | loss: 2.354343 | lrm: 1.00 | dt: 2502.03ms | tok/sec: 209,544 | bf16_mfu: 31.99 | epoch: 1 pq: 14 rg: 8 | total time: 51.03m | eta: 11.2m +step 01233/01500 (82.20%) | loss: 2.371003 | lrm: 1.00 | dt: 2497.85ms | tok/sec: 209,895 | bf16_mfu: 32.05 | epoch: 1 pq: 14 rg: 8 | total time: 51.07m | eta: 11.1m +step 01234/01500 (82.27%) | loss: 2.352386 | lrm: 1.00 | dt: 2499.29ms | tok/sec: 209,774 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 8 | total time: 51.11m | eta: 11.1m +step 01235/01500 (82.33%) | loss: 2.389660 | lrm: 1.00 | dt: 2503.97ms | tok/sec: 209,382 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 8 | total time: 51.15m | eta: 11.1m +step 01236/01500 (82.40%) | loss: 2.360726 | lrm: 1.00 | dt: 2502.77ms | tok/sec: 209,483 | bf16_mfu: 31.98 | epoch: 1 pq: 14 rg: 8 | total time: 51.19m | eta: 11.0m +step 01237/01500 (82.47%) | loss: 2.362755 | lrm: 1.00 | dt: 2497.18ms | tok/sec: 209,951 | bf16_mfu: 32.05 | epoch: 1 pq: 14 rg: 8 | total time: 51.23m | eta: 11.0m +step 01238/01500 (82.53%) | loss: 2.368698 | lrm: 1.00 | dt: 2500.44ms | tok/sec: 209,678 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 16 | total time: 51.28m | eta: 10.9m +step 01239/01500 (82.60%) | loss: 2.393627 | lrm: 1.00 | dt: 2496.97ms | tok/sec: 209,969 | bf16_mfu: 32.06 | epoch: 1 pq: 14 rg: 16 | total time: 51.32m | eta: 10.9m +step 01240/01500 (82.67%) | loss: 2.452025 | lrm: 1.00 | dt: 2499.03ms | tok/sec: 209,796 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 16 | total time: 51.36m | eta: 10.9m +step 01241/01500 (82.73%) | loss: 2.425301 | lrm: 1.00 | dt: 2503.15ms | tok/sec: 209,451 | bf16_mfu: 31.98 | epoch: 1 pq: 14 rg: 16 | total time: 51.40m | eta: 10.8m +step 01242/01500 (82.80%) | loss: 2.406693 | lrm: 1.00 | dt: 2504.33ms | tok/sec: 209,352 | bf16_mfu: 31.96 | epoch: 1 pq: 14 rg: 16 | total time: 51.44m | eta: 10.8m +step 01243/01500 (82.87%) | loss: 2.389649 | lrm: 1.00 | dt: 2503.29ms | tok/sec: 209,439 | bf16_mfu: 31.98 | epoch: 1 pq: 14 rg: 16 | total time: 51.48m | eta: 10.7m +step 01244/01500 (82.93%) | loss: 2.383802 | lrm: 1.00 | dt: 2501.70ms | tok/sec: 209,572 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 16 | total time: 51.53m | eta: 10.7m +step 01245/01500 (83.00%) | loss: 2.383217 | lrm: 1.00 | dt: 2501.24ms | tok/sec: 209,611 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 16 | total time: 51.57m | eta: 10.6m +step 01246/01500 (83.07%) | loss: 2.355744 | lrm: 1.00 | dt: 2496.78ms | tok/sec: 209,985 | bf16_mfu: 32.06 | epoch: 1 pq: 14 rg: 24 | total time: 51.61m | eta: 10.6m +step 01247/01500 (83.13%) | loss: 2.357943 | lrm: 1.00 | dt: 2506.83ms | tok/sec: 209,144 | bf16_mfu: 31.93 | epoch: 1 pq: 14 rg: 24 | total time: 51.65m | eta: 10.6m +step 01248/01500 (83.20%) | loss: 2.360747 | lrm: 1.00 | dt: 2504.34ms | tok/sec: 209,351 | bf16_mfu: 31.96 | epoch: 1 pq: 14 rg: 24 | total time: 51.69m | eta: 10.5m +step 01249/01500 (83.27%) | loss: 2.363689 | lrm: 1.00 | dt: 2499.03ms | tok/sec: 209,796 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 24 | total time: 51.73m | eta: 10.5m +<|bos|>The capital of France is Paris. The city is known for its historical landmarks, such as the Eiffel Tower +<|bos|>The chemical symbol of gold is Au. It is a soft, yellow, dense, ductile, malleable, and +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of energy, a state of motion, a state +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like to wear blue jeans and a blue shirt. I like to +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +2026-08-10 22:49:26,478 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank3.pt +2026-08-10 22:49:47,566 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank1.pt +2026-08-10 22:50:08,904 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank5.pt +2026-08-10 22:50:20,188 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank7.pt +2026-08-10 22:50:24,906 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank2.pt +2026-08-10 22:50:24,907 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank4.pt +2026-08-10 22:50:24,982 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank6.pt +2026-08-10 22:51:45,997 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_001250.pt +2026-08-10 22:51:46,001 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_001250.json +2026-08-10 22:51:54,408 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001250_rank0.pt +step 01250/01500 (83.33%) | loss: 2.361200 | lrm: 1.00 | dt: 2476.27ms | tok/sec: 211,724 | bf16_mfu: 32.32 | epoch: 1 pq: 14 rg: 24 | total time: 51.78m | eta: 10.4m +step 01251/01500 (83.40%) | loss: 2.330253 | lrm: 1.00 | dt: 2489.12ms | tok/sec: 210,631 | bf16_mfu: 32.16 | epoch: 1 pq: 14 rg: 24 | total time: 51.82m | eta: 10.4m +step 01252/01500 (83.47%) | loss: 2.327421 | lrm: 1.00 | dt: 2492.38ms | tok/sec: 210,355 | bf16_mfu: 32.12 | epoch: 1 pq: 14 rg: 24 | total time: 51.86m | eta: 10.4m +step 01253/01500 (83.53%) | loss: 2.342858 | lrm: 1.00 | dt: 2493.54ms | tok/sec: 210,258 | bf16_mfu: 32.10 | epoch: 1 pq: 14 rg: 24 | total time: 51.90m | eta: 10.3m +step 01254/01500 (83.60%) | loss: 2.348323 | lrm: 1.00 | dt: 2496.53ms | tok/sec: 210,006 | bf16_mfu: 32.06 | epoch: 1 pq: 14 rg: 32 | total time: 51.94m | eta: 10.3m +step 01255/01500 (83.67%) | loss: 2.349545 | lrm: 1.00 | dt: 2496.69ms | tok/sec: 209,993 | bf16_mfu: 32.06 | epoch: 1 pq: 14 rg: 32 | total time: 51.98m | eta: 10.2m +step 01256/01500 (83.73%) | loss: 2.331776 | lrm: 1.00 | dt: 2498.68ms | tok/sec: 209,826 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 32 | total time: 52.02m | eta: 10.2m +step 01257/01500 (83.80%) | loss: 2.300347 | lrm: 1.00 | dt: 2501.04ms | tok/sec: 209,628 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 32 | total time: 52.07m | eta: 10.1m +step 01258/01500 (83.87%) | loss: 2.317687 | lrm: 1.00 | dt: 2502.44ms | tok/sec: 209,510 | bf16_mfu: 31.99 | epoch: 1 pq: 14 rg: 32 | total time: 52.11m | eta: 10.1m +step 01259/01500 (83.93%) | loss: 2.330505 | lrm: 1.00 | dt: 2510.40ms | tok/sec: 208,846 | bf16_mfu: 31.89 | epoch: 1 pq: 14 rg: 32 | total time: 52.15m | eta: 10.1m +step 01260/01500 (84.00%) | loss: 2.323666 | lrm: 1.00 | dt: 2506.75ms | tok/sec: 209,150 | bf16_mfu: 31.93 | epoch: 1 pq: 14 rg: 32 | total time: 52.19m | eta: 10.0m +step 01261/01500 (84.07%) | loss: 2.324743 | lrm: 1.00 | dt: 2510.75ms | tok/sec: 208,817 | bf16_mfu: 31.88 | epoch: 1 pq: 14 rg: 32 | total time: 52.23m | eta: 10.0m +step 01262/01500 (84.13%) | loss: 2.364188 | lrm: 1.00 | dt: 2508.82ms | tok/sec: 208,977 | bf16_mfu: 31.91 | epoch: 1 pq: 14 rg: 40 | total time: 52.28m | eta: 9.9m +step 01263/01500 (84.20%) | loss: 2.375115 | lrm: 1.00 | dt: 2503.92ms | tok/sec: 209,386 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 40 | total time: 52.32m | eta: 9.9m +step 01264/01500 (84.27%) | loss: 2.383553 | lrm: 1.00 | dt: 2499.87ms | tok/sec: 209,726 | bf16_mfu: 32.02 | epoch: 1 pq: 14 rg: 40 | total time: 52.36m | eta: 9.9m +step 01265/01500 (84.33%) | loss: 2.397808 | lrm: 1.00 | dt: 2505.98ms | tok/sec: 209,214 | bf16_mfu: 31.94 | epoch: 1 pq: 14 rg: 40 | total time: 52.40m | eta: 9.8m +step 01266/01500 (84.40%) | loss: 2.385561 | lrm: 1.00 | dt: 2501.65ms | tok/sec: 209,577 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 40 | total time: 52.44m | eta: 9.8m +step 01267/01500 (84.47%) | loss: 2.382359 | lrm: 1.00 | dt: 2498.54ms | tok/sec: 209,837 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 40 | total time: 52.48m | eta: 9.7m +step 01268/01500 (84.53%) | loss: 2.390636 | lrm: 1.00 | dt: 2503.71ms | tok/sec: 209,404 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 40 | total time: 52.53m | eta: 9.7m +step 01269/01500 (84.60%) | loss: 2.394031 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,811 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 40 | total time: 52.57m | eta: 9.6m +step 01270/01500 (84.67%) | loss: 2.343721 | lrm: 1.00 | dt: 3570.65ms | tok/sec: 146,832 | bf16_mfu: 22.42 | epoch: 1 pq: 14 rg: 48 | total time: 52.63m | eta: 9.6m +step 01271/01500 (84.73%) | loss: 2.345929 | lrm: 1.00 | dt: 2494.67ms | tok/sec: 210,163 | bf16_mfu: 32.09 | epoch: 1 pq: 14 rg: 48 | total time: 52.67m | eta: 9.6m +step 01272/01500 (84.80%) | loss: 2.334842 | lrm: 1.00 | dt: 2494.59ms | tok/sec: 210,170 | bf16_mfu: 32.09 | epoch: 1 pq: 14 rg: 48 | total time: 52.71m | eta: 9.5m +step 01273/01500 (84.87%) | loss: 2.354259 | lrm: 1.00 | dt: 2505.26ms | tok/sec: 209,274 | bf16_mfu: 31.95 | epoch: 1 pq: 14 rg: 48 | total time: 52.75m | eta: 9.5m +step 01274/01500 (84.93%) | loss: 2.321031 | lrm: 1.00 | dt: 2498.69ms | tok/sec: 209,825 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 48 | total time: 52.79m | eta: 9.4m +step 01275/01500 (85.00%) | loss: 2.290331 | lrm: 1.00 | dt: 2501.70ms | tok/sec: 209,572 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 48 | total time: 52.83m | eta: 9.4m +step 01276/01500 (85.07%) | loss: 2.316285 | lrm: 1.00 | dt: 2497.73ms | tok/sec: 209,906 | bf16_mfu: 32.05 | epoch: 1 pq: 14 rg: 48 | total time: 52.88m | eta: 9.4m +step 01277/01500 (85.13%) | loss: 2.356199 | lrm: 1.00 | dt: 2503.38ms | tok/sec: 209,432 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 48 | total time: 52.92m | eta: 9.3m +step 01278/01500 (85.20%) | loss: 2.315412 | lrm: 1.00 | dt: 2502.47ms | tok/sec: 209,507 | bf16_mfu: 31.99 | epoch: 1 pq: 14 rg: 56 | total time: 52.96m | eta: 9.3m +step 01279/01500 (85.27%) | loss: 2.356142 | lrm: 1.00 | dt: 2500.31ms | tok/sec: 209,689 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 56 | total time: 53.00m | eta: 9.2m +step 01280/01500 (85.33%) | loss: 2.386494 | lrm: 1.00 | dt: 2498.09ms | tok/sec: 209,875 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 56 | total time: 53.04m | eta: 9.2m +step 01281/01500 (85.40%) | loss: 2.414202 | lrm: 1.00 | dt: 2498.91ms | tok/sec: 209,806 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 56 | total time: 53.08m | eta: 9.1m +step 01282/01500 (85.47%) | loss: 2.393893 | lrm: 1.00 | dt: 2500.78ms | tok/sec: 209,649 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 56 | total time: 53.13m | eta: 9.1m +step 01283/01500 (85.53%) | loss: 2.400295 | lrm: 1.00 | dt: 2500.76ms | tok/sec: 209,651 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 56 | total time: 53.17m | eta: 9.1m +step 01284/01500 (85.60%) | loss: 2.415913 | lrm: 1.00 | dt: 2498.74ms | tok/sec: 209,820 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 56 | total time: 53.21m | eta: 9.0m +step 01285/01500 (85.67%) | loss: 2.393784 | lrm: 1.00 | dt: 2500.66ms | tok/sec: 209,660 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 56 | total time: 53.25m | eta: 9.0m +step 01286/01500 (85.73%) | loss: 2.422404 | lrm: 1.00 | dt: 2507.51ms | tok/sec: 209,087 | bf16_mfu: 31.92 | epoch: 1 pq: 14 rg: 64 | total time: 53.29m | eta: 8.9m +step 01287/01500 (85.80%) | loss: 2.362867 | lrm: 1.00 | dt: 2499.79ms | tok/sec: 209,732 | bf16_mfu: 32.02 | epoch: 1 pq: 14 rg: 64 | total time: 53.33m | eta: 8.9m +step 01288/01500 (85.87%) | loss: 2.369990 | lrm: 1.00 | dt: 2502.51ms | tok/sec: 209,504 | bf16_mfu: 31.99 | epoch: 1 pq: 14 rg: 64 | total time: 53.38m | eta: 8.9m +step 01289/01500 (85.93%) | loss: 2.387106 | lrm: 1.00 | dt: 2497.97ms | tok/sec: 209,885 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 64 | total time: 53.42m | eta: 8.8m +step 01290/01500 (86.00%) | loss: 2.441367 | lrm: 1.00 | dt: 2498.72ms | tok/sec: 209,822 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 64 | total time: 53.46m | eta: 8.8m +step 01291/01500 (86.07%) | loss: 2.421063 | lrm: 1.00 | dt: 2498.61ms | tok/sec: 209,831 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 64 | total time: 53.50m | eta: 8.7m +step 01292/01500 (86.13%) | loss: 2.404441 | lrm: 1.00 | dt: 2497.75ms | tok/sec: 209,904 | bf16_mfu: 32.05 | epoch: 1 pq: 14 rg: 64 | total time: 53.54m | eta: 8.7m +step 01293/01500 (86.20%) | loss: 2.396020 | lrm: 1.00 | dt: 2498.47ms | tok/sec: 209,843 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 64 | total time: 53.58m | eta: 8.6m +step 01294/01500 (86.27%) | loss: 2.396105 | lrm: 1.00 | dt: 2498.06ms | tok/sec: 209,877 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 64 | total time: 53.63m | eta: 8.6m +step 01295/01500 (86.33%) | loss: 2.414203 | lrm: 1.00 | dt: 2499.37ms | tok/sec: 209,768 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 72 | total time: 53.67m | eta: 8.6m +step 01296/01500 (86.40%) | loss: 2.407398 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 72 | total time: 53.71m | eta: 8.5m +step 01297/01500 (86.47%) | loss: 2.449826 | lrm: 1.00 | dt: 2504.61ms | tok/sec: 209,329 | bf16_mfu: 31.96 | epoch: 1 pq: 14 rg: 72 | total time: 53.75m | eta: 8.5m +step 01298/01500 (86.53%) | loss: 2.424689 | lrm: 1.00 | dt: 2501.53ms | tok/sec: 209,586 | bf16_mfu: 32.00 | epoch: 1 pq: 14 rg: 72 | total time: 53.79m | eta: 8.4m +step 01299/01500 (86.60%) | loss: 2.431997 | lrm: 1.00 | dt: 2500.21ms | tok/sec: 209,697 | bf16_mfu: 32.02 | epoch: 1 pq: 14 rg: 72 | total time: 53.83m | eta: 8.4m +Step 01300 | Validation bpb: 0.728475 +Step 01300 | ICL early: 2.6509 late: 2.8619 score: +0.2110 | Induction loss: 9.3552 acc: 0.0929 (random-half: 14.1283) +step 01300/01500 (86.67%) | loss: 2.454086 | lrm: 1.00 | dt: 2497.64ms | tok/sec: 209,913 | bf16_mfu: 32.05 | epoch: 1 pq: 14 rg: 72 | total time: 53.88m | eta: 8.4m +step 01301/01500 (86.73%) | loss: 2.479593 | lrm: 1.00 | dt: 2500.61ms | tok/sec: 209,663 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 72 | total time: 53.92m | eta: 8.3m +step 01302/01500 (86.80%) | loss: 2.504229 | lrm: 1.00 | dt: 2498.93ms | tok/sec: 209,805 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 72 | total time: 53.96m | eta: 8.3m +step 01303/01500 (86.87%) | loss: 2.524538 | lrm: 1.00 | dt: 2500.66ms | tok/sec: 209,660 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 80 | total time: 54.00m | eta: 8.2m +step 01304/01500 (86.93%) | loss: 2.528756 | lrm: 1.00 | dt: 2498.22ms | tok/sec: 209,864 | bf16_mfu: 32.04 | epoch: 1 pq: 14 rg: 80 | total time: 54.04m | eta: 8.2m +step 01305/01500 (87.00%) | loss: 2.495952 | lrm: 1.00 | dt: 2503.92ms | tok/sec: 209,386 | bf16_mfu: 31.97 | epoch: 1 pq: 14 rg: 80 | total time: 54.08m | eta: 8.1m +step 01306/01500 (87.07%) | loss: 2.466344 | lrm: 1.00 | dt: 2505.80ms | tok/sec: 209,230 | bf16_mfu: 31.94 | epoch: 1 pq: 14 rg: 80 | total time: 54.13m | eta: 8.1m +step 01307/01500 (87.13%) | loss: 2.455489 | lrm: 1.00 | dt: 2499.37ms | tok/sec: 209,767 | bf16_mfu: 32.03 | epoch: 1 pq: 14 rg: 80 | total time: 54.17m | eta: 8.1m +step 01308/01500 (87.20%) | loss: 2.468556 | lrm: 1.00 | dt: 2506.67ms | tok/sec: 209,157 | bf16_mfu: 31.93 | epoch: 1 pq: 14 rg: 80 | total time: 54.21m | eta: 8.0m +step 01309/01500 (87.27%) | loss: 2.513764 | lrm: 1.00 | dt: 2500.46ms | tok/sec: 209,676 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 80 | total time: 54.25m | eta: 8.0m +step 01310/01500 (87.33%) | loss: 2.485792 | lrm: 1.00 | dt: 2500.41ms | tok/sec: 209,680 | bf16_mfu: 32.01 | epoch: 1 pq: 14 rg: 80 | total time: 54.29m | eta: 7.9m +step 01311/01500 (87.40%) | loss: 2.482397 | lrm: 1.00 | dt: 2499.60ms | tok/sec: 209,748 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 0 | total time: 54.34m | eta: 7.9m +step 01312/01500 (87.47%) | loss: 2.471948 | lrm: 1.00 | dt: 2499.55ms | tok/sec: 209,752 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 0 | total time: 54.38m | eta: 7.9m +step 01313/01500 (87.53%) | loss: 2.466685 | lrm: 1.00 | dt: 2500.53ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 15 rg: 0 | total time: 54.42m | eta: 7.8m +step 01314/01500 (87.60%) | loss: 2.418181 | lrm: 1.00 | dt: 2506.89ms | tok/sec: 209,138 | bf16_mfu: 31.93 | epoch: 1 pq: 15 rg: 0 | total time: 54.46m | eta: 7.8m +step 01315/01500 (87.67%) | loss: 2.433514 | lrm: 1.00 | dt: 2501.43ms | tok/sec: 209,595 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 0 | total time: 54.50m | eta: 7.7m +step 01316/01500 (87.73%) | loss: 2.432581 | lrm: 1.00 | dt: 2498.72ms | tok/sec: 209,822 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 0 | total time: 54.54m | eta: 7.7m +step 01317/01500 (87.80%) | loss: 2.454954 | lrm: 1.00 | dt: 2495.22ms | tok/sec: 210,117 | bf16_mfu: 32.08 | epoch: 1 pq: 15 rg: 0 | total time: 54.59m | eta: 7.6m +step 01318/01500 (87.87%) | loss: 2.456899 | lrm: 1.00 | dt: 2501.90ms | tok/sec: 209,556 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 0 | total time: 54.63m | eta: 7.6m +step 01319/01500 (87.93%) | loss: 2.471288 | lrm: 1.00 | dt: 2507.48ms | tok/sec: 209,089 | bf16_mfu: 31.92 | epoch: 1 pq: 15 rg: 8 | total time: 54.67m | eta: 7.6m +step 01320/01500 (88.00%) | loss: 2.497319 | lrm: 1.00 | dt: 2499.71ms | tok/sec: 209,739 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 8 | total time: 54.71m | eta: 7.5m +step 01321/01500 (88.07%) | loss: 2.521943 | lrm: 1.00 | dt: 2497.67ms | tok/sec: 209,910 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 8 | total time: 54.75m | eta: 7.5m +step 01322/01500 (88.13%) | loss: 2.468437 | lrm: 1.00 | dt: 2501.28ms | tok/sec: 209,607 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 8 | total time: 54.79m | eta: 7.4m +step 01323/01500 (88.20%) | loss: 2.497319 | lrm: 1.00 | dt: 2498.47ms | tok/sec: 209,843 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 8 | total time: 54.84m | eta: 7.4m +step 01324/01500 (88.27%) | loss: 2.500568 | lrm: 1.00 | dt: 2505.22ms | tok/sec: 209,278 | bf16_mfu: 31.95 | epoch: 1 pq: 15 rg: 8 | total time: 54.88m | eta: 7.4m +step 01325/01500 (88.33%) | loss: 2.524526 | lrm: 1.00 | dt: 2498.36ms | tok/sec: 209,853 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 8 | total time: 54.92m | eta: 7.3m +step 01326/01500 (88.40%) | loss: 2.541899 | lrm: 1.00 | dt: 2497.00ms | tok/sec: 209,966 | bf16_mfu: 32.06 | epoch: 1 pq: 15 rg: 8 | total time: 54.96m | eta: 7.3m +step 01327/01500 (88.47%) | loss: 2.579647 | lrm: 1.00 | dt: 2506.93ms | tok/sec: 209,135 | bf16_mfu: 31.93 | epoch: 1 pq: 15 rg: 16 | total time: 55.00m | eta: 7.2m +step 01328/01500 (88.53%) | loss: 2.554086 | lrm: 1.00 | dt: 2502.26ms | tok/sec: 209,525 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 16 | total time: 55.04m | eta: 7.2m +step 01329/01500 (88.60%) | loss: 2.548375 | lrm: 1.00 | dt: 2506.95ms | tok/sec: 209,133 | bf16_mfu: 31.93 | epoch: 1 pq: 15 rg: 16 | total time: 55.09m | eta: 7.1m +step 01330/01500 (88.67%) | loss: 2.498161 | lrm: 1.00 | dt: 2503.39ms | tok/sec: 209,431 | bf16_mfu: 31.97 | epoch: 1 pq: 15 rg: 16 | total time: 55.13m | eta: 7.1m +step 01331/01500 (88.73%) | loss: 2.498972 | lrm: 1.00 | dt: 2503.71ms | tok/sec: 209,404 | bf16_mfu: 31.97 | epoch: 1 pq: 15 rg: 16 | total time: 55.17m | eta: 7.1m +step 01332/01500 (88.80%) | loss: 2.508089 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 16 | total time: 55.21m | eta: 7.0m +step 01333/01500 (88.87%) | loss: 2.511404 | lrm: 1.00 | dt: 2499.18ms | tok/sec: 209,783 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 16 | total time: 55.25m | eta: 7.0m +step 01334/01500 (88.93%) | loss: 2.510940 | lrm: 1.00 | dt: 2501.09ms | tok/sec: 209,624 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 16 | total time: 55.29m | eta: 6.9m +step 01335/01500 (89.00%) | loss: 2.536113 | lrm: 1.00 | dt: 2495.40ms | tok/sec: 210,101 | bf16_mfu: 32.08 | epoch: 1 pq: 15 rg: 24 | total time: 55.34m | eta: 6.9m +step 01336/01500 (89.07%) | loss: 2.531636 | lrm: 1.00 | dt: 2505.76ms | tok/sec: 209,233 | bf16_mfu: 31.94 | epoch: 1 pq: 15 rg: 24 | total time: 55.38m | eta: 6.8m +step 01337/01500 (89.13%) | loss: 2.538024 | lrm: 1.00 | dt: 2499.30ms | tok/sec: 209,773 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 24 | total time: 55.42m | eta: 6.8m +step 01338/01500 (89.20%) | loss: 2.515331 | lrm: 1.00 | dt: 2501.75ms | tok/sec: 209,568 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 24 | total time: 55.46m | eta: 6.8m +step 01339/01500 (89.27%) | loss: 2.507357 | lrm: 1.00 | dt: 2503.18ms | tok/sec: 209,448 | bf16_mfu: 31.98 | epoch: 1 pq: 15 rg: 24 | total time: 55.50m | eta: 6.7m +step 01340/01500 (89.33%) | loss: 2.503695 | lrm: 1.00 | dt: 2495.13ms | tok/sec: 210,124 | bf16_mfu: 32.08 | epoch: 1 pq: 15 rg: 24 | total time: 55.54m | eta: 6.7m +step 01341/01500 (89.40%) | loss: 2.507368 | lrm: 1.00 | dt: 2500.79ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 15 rg: 24 | total time: 55.59m | eta: 6.6m +step 01342/01500 (89.47%) | loss: 2.531401 | lrm: 1.00 | dt: 2502.42ms | tok/sec: 209,512 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 24 | total time: 55.63m | eta: 6.6m +step 01343/01500 (89.53%) | loss: 2.550755 | lrm: 1.00 | dt: 2501.14ms | tok/sec: 209,619 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 32 | total time: 55.67m | eta: 6.6m +step 01344/01500 (89.60%) | loss: 2.541343 | lrm: 1.00 | dt: 2499.26ms | tok/sec: 209,777 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 32 | total time: 55.71m | eta: 6.5m +step 01345/01500 (89.67%) | loss: 2.531321 | lrm: 1.00 | dt: 2498.66ms | tok/sec: 209,827 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 32 | total time: 55.75m | eta: 6.5m +step 01346/01500 (89.73%) | loss: 2.550525 | lrm: 1.00 | dt: 2505.34ms | tok/sec: 209,267 | bf16_mfu: 31.95 | epoch: 1 pq: 15 rg: 32 | total time: 55.79m | eta: 6.4m +step 01347/01500 (89.80%) | loss: 2.572877 | lrm: 1.00 | dt: 2501.74ms | tok/sec: 209,569 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 32 | total time: 55.84m | eta: 6.4m +step 01348/01500 (89.87%) | loss: 2.542516 | lrm: 1.00 | dt: 2501.66ms | tok/sec: 209,575 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 32 | total time: 55.88m | eta: 6.3m +step 01349/01500 (89.93%) | loss: 2.543491 | lrm: 1.00 | dt: 2500.14ms | tok/sec: 209,703 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 32 | total time: 55.92m | eta: 6.3m +step 01350/01500 (90.00%) | loss: 2.531649 | lrm: 1.00 | dt: 2498.62ms | tok/sec: 209,830 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 40 | total time: 55.96m | eta: 6.3m +step 01351/01500 (90.07%) | loss: 2.496322 | lrm: 1.00 | dt: 3533.98ms | tok/sec: 148,356 | bf16_mfu: 22.65 | epoch: 1 pq: 15 rg: 40 | total time: 56.02m | eta: 6.2m +step 01352/01500 (90.13%) | loss: 2.514389 | lrm: 1.00 | dt: 2497.03ms | tok/sec: 209,964 | bf16_mfu: 32.06 | epoch: 1 pq: 15 rg: 40 | total time: 56.06m | eta: 6.2m +step 01353/01500 (90.20%) | loss: 2.539929 | lrm: 1.00 | dt: 2500.11ms | tok/sec: 209,706 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 40 | total time: 56.10m | eta: 6.1m +step 01354/01500 (90.27%) | loss: 2.512404 | lrm: 1.00 | dt: 2501.89ms | tok/sec: 209,556 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 40 | total time: 56.14m | eta: 6.1m +step 01355/01500 (90.33%) | loss: 2.554807 | lrm: 1.00 | dt: 2498.80ms | tok/sec: 209,816 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 40 | total time: 56.19m | eta: 6.1m +step 01356/01500 (90.40%) | loss: 2.572568 | lrm: 1.00 | dt: 2498.51ms | tok/sec: 209,840 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 40 | total time: 56.23m | eta: 6.0m +step 01357/01500 (90.47%) | loss: 2.558059 | lrm: 1.00 | dt: 2499.78ms | tok/sec: 209,733 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 40 | total time: 56.27m | eta: 6.0m +step 01358/01500 (90.53%) | loss: 2.559718 | lrm: 1.00 | dt: 2502.75ms | tok/sec: 209,484 | bf16_mfu: 31.98 | epoch: 1 pq: 15 rg: 40 | total time: 56.31m | eta: 5.9m +step 01359/01500 (90.60%) | loss: 2.585213 | lrm: 1.00 | dt: 2504.48ms | tok/sec: 209,340 | bf16_mfu: 31.96 | epoch: 1 pq: 15 rg: 48 | total time: 56.35m | eta: 5.9m +step 01360/01500 (90.67%) | loss: 2.567475 | lrm: 1.00 | dt: 2505.23ms | tok/sec: 209,277 | bf16_mfu: 31.95 | epoch: 1 pq: 15 rg: 48 | total time: 56.39m | eta: 5.8m +step 01361/01500 (90.73%) | loss: 2.560613 | lrm: 1.00 | dt: 2497.82ms | tok/sec: 209,898 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 48 | total time: 56.44m | eta: 5.8m +step 01362/01500 (90.80%) | loss: 2.522151 | lrm: 1.00 | dt: 2497.24ms | tok/sec: 209,947 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 48 | total time: 56.48m | eta: 5.8m +step 01363/01500 (90.87%) | loss: 2.515407 | lrm: 1.00 | dt: 2499.59ms | tok/sec: 209,749 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 48 | total time: 56.52m | eta: 5.7m +step 01364/01500 (90.93%) | loss: 2.463668 | lrm: 1.00 | dt: 2499.60ms | tok/sec: 209,749 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 48 | total time: 56.56m | eta: 5.7m +step 01365/01500 (91.00%) | loss: 2.453724 | lrm: 1.00 | dt: 2502.13ms | tok/sec: 209,536 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 48 | total time: 56.60m | eta: 5.6m +step 01366/01500 (91.07%) | loss: 2.412913 | lrm: 1.00 | dt: 2499.58ms | tok/sec: 209,750 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 48 | total time: 56.64m | eta: 5.6m +step 01367/01500 (91.13%) | loss: 2.417648 | lrm: 1.00 | dt: 2499.88ms | tok/sec: 209,725 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 56 | total time: 56.69m | eta: 5.6m +step 01368/01500 (91.20%) | loss: 2.410528 | lrm: 1.00 | dt: 2497.42ms | tok/sec: 209,932 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 56 | total time: 56.73m | eta: 5.5m +step 01369/01500 (91.27%) | loss: 2.407004 | lrm: 1.00 | dt: 2498.31ms | tok/sec: 209,857 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 56 | total time: 56.77m | eta: 5.5m +step 01370/01500 (91.33%) | loss: 2.421171 | lrm: 1.00 | dt: 2497.22ms | tok/sec: 209,948 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 56 | total time: 56.81m | eta: 5.4m +step 01371/01500 (91.40%) | loss: 2.404136 | lrm: 1.00 | dt: 2501.38ms | tok/sec: 209,599 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 56 | total time: 56.85m | eta: 5.4m +step 01372/01500 (91.47%) | loss: 2.425430 | lrm: 1.00 | dt: 2498.14ms | tok/sec: 209,870 | bf16_mfu: 32.04 | epoch: 1 pq: 15 rg: 56 | total time: 56.89m | eta: 5.3m +step 01373/01500 (91.53%) | loss: 2.435518 | lrm: 1.00 | dt: 2500.87ms | tok/sec: 209,642 | bf16_mfu: 32.01 | epoch: 1 pq: 15 rg: 56 | total time: 56.94m | eta: 5.3m +step 01374/01500 (91.60%) | loss: 2.433409 | lrm: 1.00 | dt: 2499.43ms | tok/sec: 209,763 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 56 | total time: 56.98m | eta: 5.3m +step 01375/01500 (91.67%) | loss: 2.415074 | lrm: 1.00 | dt: 2505.46ms | tok/sec: 209,258 | bf16_mfu: 31.95 | epoch: 1 pq: 15 rg: 64 | total time: 57.02m | eta: 5.2m +step 01376/01500 (91.73%) | loss: 2.424119 | lrm: 1.00 | dt: 2499.90ms | tok/sec: 209,723 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 64 | total time: 57.06m | eta: 5.2m +step 01377/01500 (91.80%) | loss: 2.424437 | lrm: 1.00 | dt: 2502.73ms | tok/sec: 209,486 | bf16_mfu: 31.98 | epoch: 1 pq: 15 rg: 64 | total time: 57.10m | eta: 5.1m +step 01378/01500 (91.87%) | loss: 2.426031 | lrm: 1.00 | dt: 2503.73ms | tok/sec: 209,402 | bf16_mfu: 31.97 | epoch: 1 pq: 15 rg: 64 | total time: 57.14m | eta: 5.1m +step 01379/01500 (91.93%) | loss: 2.419726 | lrm: 1.00 | dt: 2506.55ms | tok/sec: 209,167 | bf16_mfu: 31.93 | epoch: 1 pq: 15 rg: 64 | total time: 57.19m | eta: 5.1m +step 01380/01500 (92.00%) | loss: 2.448142 | lrm: 1.00 | dt: 2499.58ms | tok/sec: 209,750 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 64 | total time: 57.23m | eta: 5.0m +step 01381/01500 (92.07%) | loss: 2.399573 | lrm: 1.00 | dt: 2497.10ms | tok/sec: 209,958 | bf16_mfu: 32.06 | epoch: 1 pq: 15 rg: 64 | total time: 57.27m | eta: 5.0m +step 01382/01500 (92.13%) | loss: 2.383089 | lrm: 1.00 | dt: 2495.66ms | tok/sec: 210,079 | bf16_mfu: 32.07 | epoch: 1 pq: 15 rg: 64 | total time: 57.31m | eta: 4.9m +step 01383/01500 (92.20%) | loss: 2.388769 | lrm: 1.00 | dt: 2500.54ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 15 rg: 72 | total time: 57.35m | eta: 4.9m +step 01384/01500 (92.27%) | loss: 2.380467 | lrm: 1.00 | dt: 2497.57ms | tok/sec: 209,919 | bf16_mfu: 32.05 | epoch: 1 pq: 15 rg: 72 | total time: 57.39m | eta: 4.8m +step 01385/01500 (92.33%) | loss: 2.401750 | lrm: 1.00 | dt: 2501.86ms | tok/sec: 209,558 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 72 | total time: 57.44m | eta: 4.8m +step 01386/01500 (92.40%) | loss: 2.434829 | lrm: 1.00 | dt: 2504.21ms | tok/sec: 209,362 | bf16_mfu: 31.96 | epoch: 1 pq: 15 rg: 72 | total time: 57.48m | eta: 4.8m +step 01387/01500 (92.47%) | loss: 2.422441 | lrm: 1.00 | dt: 2496.20ms | tok/sec: 210,034 | bf16_mfu: 32.07 | epoch: 1 pq: 15 rg: 72 | total time: 57.52m | eta: 4.7m +step 01388/01500 (92.53%) | loss: 2.441884 | lrm: 1.00 | dt: 2501.04ms | tok/sec: 209,627 | bf16_mfu: 32.00 | epoch: 1 pq: 15 rg: 72 | total time: 57.56m | eta: 4.7m +step 01389/01500 (92.60%) | loss: 2.432226 | lrm: 1.00 | dt: 2499.63ms | tok/sec: 209,745 | bf16_mfu: 32.02 | epoch: 1 pq: 15 rg: 72 | total time: 57.60m | eta: 4.6m +step 01390/01500 (92.67%) | loss: 2.427714 | lrm: 1.00 | dt: 2504.70ms | tok/sec: 209,321 | bf16_mfu: 31.96 | epoch: 1 pq: 15 rg: 72 | total time: 57.65m | eta: 4.6m +step 01391/01500 (92.73%) | loss: 2.449333 | lrm: 1.00 | dt: 2505.90ms | tok/sec: 209,221 | bf16_mfu: 31.94 | epoch: 1 pq: 15 rg: 80 | total time: 57.69m | eta: 4.6m +step 01392/01500 (92.80%) | loss: 2.421657 | lrm: 1.00 | dt: 2503.30ms | tok/sec: 209,438 | bf16_mfu: 31.98 | epoch: 1 pq: 15 rg: 80 | total time: 57.73m | eta: 4.5m +step 01393/01500 (92.87%) | loss: 2.388789 | lrm: 1.00 | dt: 2502.54ms | tok/sec: 209,502 | bf16_mfu: 31.99 | epoch: 1 pq: 15 rg: 80 | total time: 57.77m | eta: 4.5m +step 01394/01500 (92.93%) | loss: 2.394633 | lrm: 1.00 | dt: 2495.98ms | tok/sec: 210,053 | bf16_mfu: 32.07 | epoch: 1 pq: 15 rg: 80 | total time: 57.81m | eta: 4.4m +step 01395/01500 (93.00%) | loss: 2.400391 | lrm: 1.00 | dt: 2503.12ms | tok/sec: 209,453 | bf16_mfu: 31.98 | epoch: 1 pq: 15 rg: 80 | total time: 57.85m | eta: 4.4m +step 01396/01500 (93.07%) | loss: 2.408024 | lrm: 1.00 | dt: 2505.66ms | tok/sec: 209,241 | bf16_mfu: 31.95 | epoch: 1 pq: 15 rg: 80 | total time: 57.90m | eta: 4.3m +step 01397/01500 (93.13%) | loss: 2.436805 | lrm: 1.00 | dt: 2496.85ms | tok/sec: 209,979 | bf16_mfu: 32.06 | epoch: 1 pq: 15 rg: 80 | total time: 57.94m | eta: 4.3m +step 01398/01500 (93.20%) | loss: 2.449819 | lrm: 1.00 | dt: 2498.86ms | tok/sec: 209,810 | bf16_mfu: 32.03 | epoch: 1 pq: 15 rg: 80 | total time: 57.98m | eta: 4.3m +step 01399/01500 (93.27%) | loss: 2.409758 | lrm: 1.00 | dt: 2498.56ms | tok/sec: 209,836 | bf16_mfu: 32.04 | epoch: 1 pq: 16 rg: 0 | total time: 58.02m | eta: 4.2m +Step 01400 | Validation bpb: 0.726742 +Step 01400 | ICL early: 2.6447 late: 2.8352 score: +0.1905 | Induction loss: 9.4753 acc: 0.0817 (random-half: 14.0049) +step 01400/01500 (93.33%) | loss: 2.407156 | lrm: 1.00 | dt: 2496.82ms | tok/sec: 209,981 | bf16_mfu: 32.06 | epoch: 1 pq: 16 rg: 0 | total time: 58.06m | eta: 4.2m +step 01401/01500 (93.40%) | loss: 2.403219 | lrm: 1.00 | dt: 2495.31ms | tok/sec: 210,109 | bf16_mfu: 32.08 | epoch: 1 pq: 16 rg: 0 | total time: 58.10m | eta: 4.1m +step 01402/01500 (93.47%) | loss: 2.389298 | lrm: 1.00 | dt: 2502.02ms | tok/sec: 209,545 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 0 | total time: 58.15m | eta: 4.1m +step 01403/01500 (93.53%) | loss: 2.374202 | lrm: 1.00 | dt: 2506.42ms | tok/sec: 209,178 | bf16_mfu: 31.94 | epoch: 1 pq: 16 rg: 0 | total time: 58.19m | eta: 4.1m +step 01404/01500 (93.60%) | loss: 2.410423 | lrm: 1.00 | dt: 2495.39ms | tok/sec: 210,102 | bf16_mfu: 32.08 | epoch: 1 pq: 16 rg: 0 | total time: 58.23m | eta: 4.0m +step 01405/01500 (93.67%) | loss: 2.426619 | lrm: 1.00 | dt: 2500.29ms | tok/sec: 209,691 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 0 | total time: 58.27m | eta: 4.0m +step 01406/01500 (93.73%) | loss: 2.397991 | lrm: 1.00 | dt: 2498.97ms | tok/sec: 209,801 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 0 | total time: 58.31m | eta: 3.9m +step 01407/01500 (93.80%) | loss: 2.382572 | lrm: 1.00 | dt: 2501.25ms | tok/sec: 209,610 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 8 | total time: 58.35m | eta: 3.9m +step 01408/01500 (93.87%) | loss: 2.413051 | lrm: 1.00 | dt: 2500.11ms | tok/sec: 209,706 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 8 | total time: 58.40m | eta: 3.8m +step 01409/01500 (93.93%) | loss: 2.382012 | lrm: 1.00 | dt: 2502.67ms | tok/sec: 209,491 | bf16_mfu: 31.98 | epoch: 1 pq: 16 rg: 8 | total time: 58.44m | eta: 3.8m +step 01410/01500 (94.00%) | loss: 2.365499 | lrm: 1.00 | dt: 2497.09ms | tok/sec: 209,959 | bf16_mfu: 32.06 | epoch: 1 pq: 16 rg: 8 | total time: 58.48m | eta: 3.8m +step 01411/01500 (94.07%) | loss: 2.380447 | lrm: 1.00 | dt: 2506.13ms | tok/sec: 209,202 | bf16_mfu: 31.94 | epoch: 1 pq: 16 rg: 8 | total time: 58.52m | eta: 3.7m +step 01412/01500 (94.13%) | loss: 2.356123 | lrm: 1.00 | dt: 2501.11ms | tok/sec: 209,621 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 8 | total time: 58.56m | eta: 3.7m +step 01413/01500 (94.20%) | loss: 2.395847 | lrm: 1.00 | dt: 2504.82ms | tok/sec: 209,311 | bf16_mfu: 31.96 | epoch: 1 pq: 16 rg: 8 | total time: 58.60m | eta: 3.6m +step 01414/01500 (94.27%) | loss: 2.478718 | lrm: 1.00 | dt: 2503.01ms | tok/sec: 209,463 | bf16_mfu: 31.98 | epoch: 1 pq: 16 rg: 8 | total time: 58.65m | eta: 3.6m +step 01415/01500 (94.33%) | loss: 2.453347 | lrm: 1.00 | dt: 2501.11ms | tok/sec: 209,621 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 16 | total time: 58.69m | eta: 3.6m +step 01416/01500 (94.40%) | loss: 2.473875 | lrm: 1.00 | dt: 2505.13ms | tok/sec: 209,285 | bf16_mfu: 31.95 | epoch: 1 pq: 16 rg: 16 | total time: 58.73m | eta: 3.5m +step 01417/01500 (94.47%) | loss: 2.475766 | lrm: 1.00 | dt: 2501.04ms | tok/sec: 209,627 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 16 | total time: 58.77m | eta: 3.5m +step 01418/01500 (94.53%) | loss: 2.443631 | lrm: 1.00 | dt: 2499.46ms | tok/sec: 209,760 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 16 | total time: 58.81m | eta: 3.4m +step 01419/01500 (94.60%) | loss: 2.397742 | lrm: 1.00 | dt: 2499.47ms | tok/sec: 209,759 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 16 | total time: 58.85m | eta: 3.4m +step 01420/01500 (94.67%) | loss: 2.433936 | lrm: 1.00 | dt: 2496.71ms | tok/sec: 209,991 | bf16_mfu: 32.06 | epoch: 1 pq: 16 rg: 16 | total time: 58.90m | eta: 3.3m +step 01421/01500 (94.73%) | loss: 2.418765 | lrm: 1.00 | dt: 2502.30ms | tok/sec: 209,522 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 16 | total time: 58.94m | eta: 3.3m +step 01422/01500 (94.80%) | loss: 2.397046 | lrm: 1.00 | dt: 2497.44ms | tok/sec: 209,929 | bf16_mfu: 32.05 | epoch: 1 pq: 16 rg: 16 | total time: 58.98m | eta: 3.3m +step 01423/01500 (94.87%) | loss: 2.391084 | lrm: 1.00 | dt: 2499.23ms | tok/sec: 209,779 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 24 | total time: 59.02m | eta: 3.2m +step 01424/01500 (94.93%) | loss: 2.398878 | lrm: 1.00 | dt: 2498.95ms | tok/sec: 209,803 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 24 | total time: 59.06m | eta: 3.2m +step 01425/01500 (95.00%) | loss: 2.419595 | lrm: 1.00 | dt: 2499.91ms | tok/sec: 209,723 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 24 | total time: 59.10m | eta: 3.1m +step 01426/01500 (95.07%) | loss: 2.457590 | lrm: 1.00 | dt: 2502.04ms | tok/sec: 209,544 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 24 | total time: 59.15m | eta: 3.1m +step 01427/01500 (95.13%) | loss: 2.493086 | lrm: 1.00 | dt: 2502.18ms | tok/sec: 209,532 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 24 | total time: 59.19m | eta: 3.0m +step 01428/01500 (95.20%) | loss: 2.485929 | lrm: 1.00 | dt: 2498.83ms | tok/sec: 209,812 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 24 | total time: 59.23m | eta: 3.0m +step 01429/01500 (95.27%) | loss: 2.487202 | lrm: 1.00 | dt: 2499.08ms | tok/sec: 209,792 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 24 | total time: 59.27m | eta: 3.0m +step 01430/01500 (95.33%) | loss: 2.471029 | lrm: 1.00 | dt: 2502.32ms | tok/sec: 209,520 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 24 | total time: 59.31m | eta: 2.9m +step 01431/01500 (95.40%) | loss: 2.456697 | lrm: 1.00 | dt: 2505.03ms | tok/sec: 209,294 | bf16_mfu: 31.95 | epoch: 1 pq: 16 rg: 24 | total time: 59.35m | eta: 2.9m +step 01432/01500 (95.47%) | loss: 2.437899 | lrm: 1.00 | dt: 2747.31ms | tok/sec: 190,836 | bf16_mfu: 29.14 | epoch: 1 pq: 16 rg: 32 | total time: 59.40m | eta: 2.8m +step 01433/01500 (95.53%) | loss: 2.431286 | lrm: 1.00 | dt: 2500.94ms | tok/sec: 209,636 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 32 | total time: 59.44m | eta: 2.8m +step 01434/01500 (95.60%) | loss: 2.451353 | lrm: 1.00 | dt: 2500.11ms | tok/sec: 209,706 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 32 | total time: 59.48m | eta: 2.8m +step 01435/01500 (95.67%) | loss: 2.444935 | lrm: 1.00 | dt: 2507.48ms | tok/sec: 209,089 | bf16_mfu: 31.92 | epoch: 1 pq: 16 rg: 32 | total time: 59.52m | eta: 2.7m +step 01436/01500 (95.73%) | loss: 2.460388 | lrm: 1.00 | dt: 2500.79ms | tok/sec: 209,648 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 32 | total time: 59.57m | eta: 2.7m +step 01437/01500 (95.80%) | loss: 2.473801 | lrm: 1.00 | dt: 2506.70ms | tok/sec: 209,154 | bf16_mfu: 31.93 | epoch: 1 pq: 16 rg: 32 | total time: 59.61m | eta: 2.6m +step 01438/01500 (95.87%) | loss: 2.483231 | lrm: 1.00 | dt: 2501.54ms | tok/sec: 209,586 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 32 | total time: 59.65m | eta: 2.6m +step 01439/01500 (95.93%) | loss: 2.443377 | lrm: 1.00 | dt: 2502.14ms | tok/sec: 209,536 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 32 | total time: 59.69m | eta: 2.5m +step 01440/01500 (96.00%) | loss: 2.462744 | lrm: 1.00 | dt: 2504.65ms | tok/sec: 209,325 | bf16_mfu: 31.96 | epoch: 1 pq: 16 rg: 40 | total time: 59.73m | eta: 2.5m +step 01441/01500 (96.07%) | loss: 2.448690 | lrm: 1.00 | dt: 2502.88ms | tok/sec: 209,473 | bf16_mfu: 31.98 | epoch: 1 pq: 16 rg: 40 | total time: 59.78m | eta: 2.5m +step 01442/01500 (96.13%) | loss: 2.443707 | lrm: 1.00 | dt: 2497.90ms | tok/sec: 209,891 | bf16_mfu: 32.05 | epoch: 1 pq: 16 rg: 40 | total time: 59.82m | eta: 2.4m +step 01443/01500 (96.20%) | loss: 2.447738 | lrm: 1.00 | dt: 2503.93ms | tok/sec: 209,386 | bf16_mfu: 31.97 | epoch: 1 pq: 16 rg: 40 | total time: 59.86m | eta: 2.4m +step 01444/01500 (96.27%) | loss: 2.430013 | lrm: 1.00 | dt: 2498.28ms | tok/sec: 209,859 | bf16_mfu: 32.04 | epoch: 1 pq: 16 rg: 40 | total time: 59.90m | eta: 2.3m +step 01445/01500 (96.33%) | loss: 2.428550 | lrm: 1.00 | dt: 2503.70ms | tok/sec: 209,405 | bf16_mfu: 31.97 | epoch: 1 pq: 16 rg: 40 | total time: 59.94m | eta: 2.3m +step 01446/01500 (96.40%) | loss: 2.428568 | lrm: 1.00 | dt: 2501.13ms | tok/sec: 209,620 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 40 | total time: 59.98m | eta: 2.3m +step 01447/01500 (96.47%) | loss: 2.453339 | lrm: 1.00 | dt: 2501.42ms | tok/sec: 209,596 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 40 | total time: 60.03m | eta: 2.2m +step 01448/01500 (96.53%) | loss: 2.472597 | lrm: 1.00 | dt: 2503.40ms | tok/sec: 209,430 | bf16_mfu: 31.97 | epoch: 1 pq: 16 rg: 48 | total time: 60.07m | eta: 2.2m +step 01449/01500 (96.60%) | loss: 2.449282 | lrm: 1.00 | dt: 2499.85ms | tok/sec: 209,727 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 48 | total time: 60.11m | eta: 2.1m +step 01450/01500 (96.67%) | loss: 2.456129 | lrm: 1.00 | dt: 2505.50ms | tok/sec: 209,254 | bf16_mfu: 31.95 | epoch: 1 pq: 16 rg: 48 | total time: 60.15m | eta: 2.1m +step 01451/01500 (96.73%) | loss: 2.478262 | lrm: 1.00 | dt: 2495.35ms | tok/sec: 210,105 | bf16_mfu: 32.08 | epoch: 1 pq: 16 rg: 48 | total time: 60.19m | eta: 2.0m +step 01452/01500 (96.80%) | loss: 2.469788 | lrm: 1.00 | dt: 2500.53ms | tok/sec: 209,670 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 48 | total time: 60.23m | eta: 2.0m +step 01453/01500 (96.87%) | loss: 2.490518 | lrm: 1.00 | dt: 2507.76ms | tok/sec: 209,066 | bf16_mfu: 31.92 | epoch: 1 pq: 16 rg: 48 | total time: 60.28m | eta: 2.0m +step 01454/01500 (96.93%) | loss: 2.480713 | lrm: 1.00 | dt: 2501.42ms | tok/sec: 209,595 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 48 | total time: 60.32m | eta: 1.9m +step 01455/01500 (97.00%) | loss: 2.483244 | lrm: 1.00 | dt: 2500.85ms | tok/sec: 209,643 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 48 | total time: 60.36m | eta: 1.9m +step 01456/01500 (97.07%) | loss: 2.475816 | lrm: 1.00 | dt: 2499.12ms | tok/sec: 209,789 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 56 | total time: 60.40m | eta: 1.8m +step 01457/01500 (97.13%) | loss: 2.505269 | lrm: 1.00 | dt: 2502.45ms | tok/sec: 209,509 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 56 | total time: 60.44m | eta: 1.8m +step 01458/01500 (97.20%) | loss: 2.408040 | lrm: 1.00 | dt: 2500.37ms | tok/sec: 209,684 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 56 | total time: 60.48m | eta: 1.8m +step 01459/01500 (97.27%) | loss: 2.404778 | lrm: 1.00 | dt: 2501.87ms | tok/sec: 209,558 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 56 | total time: 60.53m | eta: 1.7m +step 01460/01500 (97.33%) | loss: 2.401158 | lrm: 1.00 | dt: 2505.31ms | tok/sec: 209,270 | bf16_mfu: 31.95 | epoch: 1 pq: 16 rg: 56 | total time: 60.57m | eta: 1.7m +step 01461/01500 (97.40%) | loss: 2.393260 | lrm: 1.00 | dt: 2499.41ms | tok/sec: 209,764 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 56 | total time: 60.61m | eta: 1.6m +step 01462/01500 (97.47%) | loss: 2.377952 | lrm: 1.00 | dt: 2501.03ms | tok/sec: 209,628 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 56 | total time: 60.65m | eta: 1.6m +step 01463/01500 (97.53%) | loss: 2.363887 | lrm: 1.00 | dt: 2497.56ms | tok/sec: 209,920 | bf16_mfu: 32.05 | epoch: 1 pq: 16 rg: 56 | total time: 60.69m | eta: 1.5m +step 01464/01500 (97.60%) | loss: 2.300430 | lrm: 1.00 | dt: 2496.16ms | tok/sec: 210,038 | bf16_mfu: 32.07 | epoch: 1 pq: 16 rg: 64 | total time: 60.73m | eta: 1.5m +step 01465/01500 (97.67%) | loss: 2.296305 | lrm: 1.00 | dt: 2504.82ms | tok/sec: 209,311 | bf16_mfu: 31.96 | epoch: 1 pq: 16 rg: 64 | total time: 60.78m | eta: 1.5m +step 01466/01500 (97.73%) | loss: 2.294729 | lrm: 1.00 | dt: 2505.22ms | tok/sec: 209,278 | bf16_mfu: 31.95 | epoch: 1 pq: 16 rg: 64 | total time: 60.82m | eta: 1.4m +step 01467/01500 (97.80%) | loss: 2.273269 | lrm: 1.00 | dt: 2500.66ms | tok/sec: 209,659 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 64 | total time: 60.86m | eta: 1.4m +step 01468/01500 (97.87%) | loss: 2.291700 | lrm: 1.00 | dt: 2498.76ms | tok/sec: 209,819 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 64 | total time: 60.90m | eta: 1.3m +step 01469/01500 (97.93%) | loss: 2.234252 | lrm: 1.00 | dt: 2499.27ms | tok/sec: 209,776 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 64 | total time: 60.94m | eta: 1.3m +step 01470/01500 (98.00%) | loss: 2.249146 | lrm: 1.00 | dt: 2503.34ms | tok/sec: 209,435 | bf16_mfu: 31.98 | epoch: 1 pq: 16 rg: 64 | total time: 60.98m | eta: 1.3m +step 01471/01500 (98.07%) | loss: 2.286775 | lrm: 1.00 | dt: 2504.94ms | tok/sec: 209,301 | bf16_mfu: 31.96 | epoch: 1 pq: 16 rg: 64 | total time: 61.03m | eta: 1.2m +step 01472/01500 (98.13%) | loss: 2.296955 | lrm: 1.00 | dt: 2497.88ms | tok/sec: 209,892 | bf16_mfu: 32.05 | epoch: 1 pq: 16 rg: 64 | total time: 61.07m | eta: 1.2m +step 01473/01500 (98.20%) | loss: 2.349969 | lrm: 1.00 | dt: 2500.89ms | tok/sec: 209,640 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 72 | total time: 61.11m | eta: 1.1m +step 01474/01500 (98.27%) | loss: 2.382145 | lrm: 1.00 | dt: 2502.49ms | tok/sec: 209,506 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 72 | total time: 61.15m | eta: 1.1m +step 01475/01500 (98.33%) | loss: 2.374149 | lrm: 1.00 | dt: 2497.38ms | tok/sec: 209,935 | bf16_mfu: 32.05 | epoch: 1 pq: 16 rg: 72 | total time: 61.19m | eta: 1.0m +step 01476/01500 (98.40%) | loss: 2.375506 | lrm: 1.00 | dt: 2501.69ms | tok/sec: 209,573 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 72 | total time: 61.23m | eta: 1.0m +step 01477/01500 (98.47%) | loss: 2.387930 | lrm: 1.00 | dt: 2501.36ms | tok/sec: 209,601 | bf16_mfu: 32.00 | epoch: 1 pq: 16 rg: 72 | total time: 61.28m | eta: 1.0m +step 01478/01500 (98.53%) | loss: 2.390028 | lrm: 1.00 | dt: 2499.69ms | tok/sec: 209,741 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 72 | total time: 61.32m | eta: 0.9m +step 01479/01500 (98.60%) | loss: 2.376601 | lrm: 1.00 | dt: 2499.56ms | tok/sec: 209,752 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 72 | total time: 61.36m | eta: 0.9m +step 01480/01500 (98.67%) | loss: 2.330563 | lrm: 1.00 | dt: 2504.31ms | tok/sec: 209,353 | bf16_mfu: 31.96 | epoch: 1 pq: 16 rg: 72 | total time: 61.40m | eta: 0.8m +step 01481/01500 (98.73%) | loss: 2.326115 | lrm: 1.00 | dt: 2498.09ms | tok/sec: 209,875 | bf16_mfu: 32.04 | epoch: 1 pq: 16 rg: 80 | total time: 61.44m | eta: 0.8m +step 01482/01500 (98.80%) | loss: 2.331603 | lrm: 1.00 | dt: 2500.58ms | tok/sec: 209,666 | bf16_mfu: 32.01 | epoch: 1 pq: 16 rg: 80 | total time: 61.48m | eta: 0.8m +step 01483/01500 (98.87%) | loss: 2.368139 | lrm: 1.00 | dt: 2503.89ms | tok/sec: 209,389 | bf16_mfu: 31.97 | epoch: 1 pq: 16 rg: 80 | total time: 61.53m | eta: 0.7m +step 01484/01500 (98.93%) | loss: 2.375422 | lrm: 1.00 | dt: 2498.38ms | tok/sec: 209,850 | bf16_mfu: 32.04 | epoch: 1 pq: 16 rg: 80 | total time: 61.57m | eta: 0.7m +step 01485/01500 (99.00%) | loss: 2.354445 | lrm: 1.00 | dt: 2502.87ms | tok/sec: 209,475 | bf16_mfu: 31.98 | epoch: 1 pq: 16 rg: 80 | total time: 61.61m | eta: 0.6m +step 01486/01500 (99.07%) | loss: 2.357608 | lrm: 1.00 | dt: 2499.33ms | tok/sec: 209,771 | bf16_mfu: 32.03 | epoch: 1 pq: 16 rg: 80 | total time: 61.65m | eta: 0.6m +step 01487/01500 (99.13%) | loss: 2.380119 | lrm: 1.00 | dt: 2502.01ms | tok/sec: 209,546 | bf16_mfu: 31.99 | epoch: 1 pq: 16 rg: 80 | total time: 61.69m | eta: 0.5m +step 01488/01500 (99.20%) | loss: 2.400317 | lrm: 1.00 | dt: 2500.08ms | tok/sec: 209,708 | bf16_mfu: 32.02 | epoch: 1 pq: 16 rg: 80 | total time: 61.73m | eta: 0.5m +step 01489/01500 (99.27%) | loss: 2.360456 | lrm: 1.00 | dt: 2500.68ms | tok/sec: 209,658 | bf16_mfu: 32.01 | epoch: 1 pq: 17 rg: 0 | total time: 61.78m | eta: 0.5m +step 01490/01500 (99.33%) | loss: 2.400154 | lrm: 1.00 | dt: 2504.74ms | tok/sec: 209,317 | bf16_mfu: 31.96 | epoch: 1 pq: 17 rg: 0 | total time: 61.82m | eta: 0.4m +step 01491/01500 (99.40%) | loss: 2.391098 | lrm: 1.00 | dt: 2508.42ms | tok/sec: 209,011 | bf16_mfu: 31.91 | epoch: 1 pq: 17 rg: 0 | total time: 61.86m | eta: 0.4m +step 01492/01500 (99.47%) | loss: 2.382892 | lrm: 1.00 | dt: 2498.70ms | tok/sec: 209,824 | bf16_mfu: 32.03 | epoch: 1 pq: 17 rg: 0 | total time: 61.90m | eta: 0.3m +step 01493/01500 (99.53%) | loss: 2.386637 | lrm: 1.00 | dt: 2501.30ms | tok/sec: 209,606 | bf16_mfu: 32.00 | epoch: 1 pq: 17 rg: 0 | total time: 61.94m | eta: 0.3m +step 01494/01500 (99.60%) | loss: 2.416373 | lrm: 1.00 | dt: 2503.63ms | tok/sec: 209,411 | bf16_mfu: 31.97 | epoch: 1 pq: 17 rg: 0 | total time: 61.98m | eta: 0.3m +step 01495/01500 (99.67%) | loss: 2.397640 | lrm: 1.00 | dt: 2505.17ms | tok/sec: 209,282 | bf16_mfu: 31.95 | epoch: 1 pq: 17 rg: 0 | total time: 62.03m | eta: 0.2m +step 01496/01500 (99.73%) | loss: 2.419714 | lrm: 1.00 | dt: 2499.40ms | tok/sec: 209,765 | bf16_mfu: 32.03 | epoch: 1 pq: 17 rg: 0 | total time: 62.07m | eta: 0.2m +step 01497/01500 (99.80%) | loss: 2.435781 | lrm: 1.00 | dt: 2501.58ms | tok/sec: 209,582 | bf16_mfu: 32.00 | epoch: 1 pq: 17 rg: 8 | total time: 62.11m | eta: 0.1m +step 01498/01500 (99.87%) | loss: 2.414833 | lrm: 1.00 | dt: 2502.76ms | tok/sec: 209,483 | bf16_mfu: 31.98 | epoch: 1 pq: 17 rg: 8 | total time: 62.15m | eta: 0.1m +step 01499/01500 (99.93%) | loss: 2.365202 | lrm: 1.00 | dt: 2500.78ms | tok/sec: 209,649 | bf16_mfu: 32.01 | epoch: 1 pq: 17 rg: 8 | total time: 62.19m | eta: 0.0m +Step 01500 | Validation bpb: 0.725184 +Step 01500 | ICL early: 2.6255 late: 2.8190 score: +0.1936 | Induction loss: 9.1172 acc: 0.1010 (random-half: 14.1163) +<|bos|>The capital of France is Paris. The capital of France is Paris. The capital of France is Paris. +<|bos|>The chemical symbol of gold is Au. It is a soft, yellow, lustrous metal. It is one +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky, the ocean, and the sky. I love +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +2026-08-10 23:12:52,520 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank3.pt +2026-08-10 23:13:13,943 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank7.pt +2026-08-10 23:13:27,026 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank4.pt +2026-08-10 23:13:31,204 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank1.pt +2026-08-10 23:13:33,896 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank6.pt +2026-08-10 23:13:34,565 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank5.pt +2026-08-10 23:15:27,234 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank2.pt +2026-08-10 23:15:33,909 - nanochat.checkpoint_manager - INFO - Saved model parameters to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/model_001500.pt +2026-08-10 23:15:33,913 - nanochat.checkpoint_manager - INFO - Saved metadata to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/meta_001500.json +2026-08-10 23:15:42,503 - nanochat.checkpoint_manager - INFO - Saved optimizer state to: /root/.cache/nanochat-d32ft/base_checkpoints/d32cmap/optim_001500_rank0.pt +Peak memory usage: 60713.63MiB +Total training time: 62.19m +Minimum validation bpb: 0.725184 +[W810 23:15:45.815974576 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator()) diff --git a/samples/d32cmap_samples.txt b/samples/d32cmap_samples.txt new file mode 100644 index 0000000000000000000000000000000000000000..ad198f6b33e4125a48c5f77da7c4e8b4740e5086 --- /dev/null +++ b/samples/d32cmap_samples.txt @@ -0,0 +1,43 @@ +# generation samples from run d32cmap +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au, and the atomic number of gold is 10. The atomic number of +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today is Saturday, tomorrow will be Sunday. If today is Sunday +<|bos|>The opposite of hot is cold. Hot is a temperature that is above the boiling point of water. Cold +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like blue because it makes me feel happy. I like blue because +<|bos|>If 5*x + 3 = 13, then x is 3. If x = 3, then x = 3. If +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au. The chemical symbol of silver is Ag. The chemical symbol of copper is +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. Today is Saturday, and tomorrow is Sunday. If today is Saturday, +<|bos|>The opposite of hot is cold. Hot is a color that is associated with the sun and the sun's +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like to wear blue jeans and I like to wear blue jeans. +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +<|bos|>The capital of France is Paris. The city is the capital of France and is the capital of the Î +<|bos|>The chemical symbol of gold is Au. It is a chemical element with the symbol Au. It is a chemical +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky and the ocean. I love the ocean because it +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +<|bos|>The capital of France is Paris. It is the capital of the Île-de-France region. It +<|bos|>The chemical symbol of gold is Au. It is a chemical element with the symbol Au. It is a chemical +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. Today is the 14th day of the month. The 14 +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky, the ocean, and the sky. I love +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +<|bos|>The capital of France is Paris. The city is known for its historical landmarks, such as the Eiffel Tower +<|bos|>The chemical symbol of gold is Au. It is a soft, yellow, dense, ductile, malleable, and +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of energy, a state of motion, a state +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I like to wear blue jeans and a blue shirt. I like to +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then +<|bos|>The capital of France is Paris. The capital of France is Paris. The capital of France is Paris. +<|bos|>The chemical symbol of gold is Au. It is a soft, yellow, lustrous metal. It is one +<|bos|>If yesterday was Friday, then tomorrow will be Saturday. If today was Saturday, then tomorrow will be Sunday. If today was +<|bos|>The opposite of hot is cold. Hot is a state of high temperature, while cold is a state of +<|bos|>The planets of the solar system are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune, +<|bos|>My favorite color is blue. I love the sky, the ocean, and the sky. I love +<|bos|>If 5*x + 3 = 13, then x is 3. If 5*x + 3 = 13, then